[Zope-Checkins] CVS: Zope2 - testCatalog.py:1.1.2.19
Digicool CVS
cvs@digicool.com
Wed, 7 Mar 2001 14:07:35 -0500
Update of /mnt/cvs-repository/Zope2/lib/python/Products/ZCatalog/tests
In directory yetix:/work/Zope2/CatalogUnit/lib/python/Products/ZCatalog/tests
Modified Files:
Tag: Catalog-Unittest-Branch
testCatalog.py
Log Message:
Code cleanup
--- Updated File testCatalog.py in package Zope2 --
--- testCatalog.py 2001/03/07 12:46:32 1.1.2.18
+++ testCatalog.py 2001/03/07 14:07:51 1.1.2.19
@@ -7,6 +7,9 @@
Andreas Jung, andreas@digicool.com
$Log$
+ Revision 1.1.2.19 2001/03/07 14:07:51 andreas
+ Code cleanup
+
Revision 1.1.2.18 2001/03/07 12:46:32 andreas
added advanced tests
@@ -18,52 +21,6 @@
- tests must now be started in the lib/python directory
- older input sets are no longer valid (must be recreated)
- Revision 1.1.2.15 2001/03/02 17:03:03 andreas
- changed default settings
-
- Revision 1.1.2.14 2001/03/02 15:16:47 andreas
- version for release
-
- Revision 1.1.2.13 2001/03/02 00:41:33 andreas
- SHould now be a "final" version
-
- Revision 1.1.2.12 2001/03/01 23:46:16 andreas
- complete thread handling rewrite
-
- Revision 1.1.2.11 2001/03/01 18:35:50 andreas
- simple tests are now doing benchmarks
-
- Revision 1.1.2.10 2001/02/28 20:23:23 andreas
- minor changes
-
- Revision 1.1.2.9 2001/02/28 18:39:20 andreas
- misc changes
-
- Revision 1.1.2.8 2001/02/28 16:51:32 andreas
- added benchmarks
-
- Revision 1.1.2.7 2001/02/28 16:02:15 andreas
- fixed bug in generation of keywords index
-
- Revision 1.1.2.6 2001/02/28 15:31:19 andreas
- updated tests
-
- Revision 1.1.2.5 2001/02/27 21:06:18 andreas
- minor changes
-
- Revision 1.1.2.4 2001/02/27 20:43:08 andreas
- added -d option
-
- Revision 1.1.2.3 2001/02/27 20:26:24 andreas
- added prelimary stress test
-
- Revision 1.1.2.2 2001/02/27 19:33:55 andreas
- detabbed version
-
- Revision 1.1.2.1 2001/02/27 19:27:58 andreas
- first lame version
-
-
"""
import os,sys
@@ -82,13 +39,8 @@
import Zope
import ZODB, ZODB.FileStorage
from Products.ZCatalog import Catalog,Vocabulary
-from SearchIndex.UnIndex import UnIndex
-from SearchIndex.UnTextIndex import UnTextIndex
-from SearchIndex.UnKeywordIndex import UnKeywordIndex
-from SearchIndex.Lexicon import Lexicon, stop_word_dict
import Persistence
import ExtensionClass
-import DateTime
import dispatcher,keywords
from zLOG import LOG
@@ -102,8 +54,6 @@
# maximum number of threads for stress testa
numThreads = 4
-# dictionary with test words
-dictFile = "/usr/share/dict/words"
# number of iterations for searches
searchIterations = 1000
@@ -120,36 +70,7 @@
# Don't change anything below
#
-def myLOG(*args):
- args = map(str,args)
- LOG('catalog',0,'bench', string.join(args , ' '))
- open('bench.log','a').write( string.join(args," ") + "\n")
-
-
-class Timer:
-
- def __init__(self,name=''):
- self.name = name
- self.start()
- def start(self):
- self.ts = time.time()
-
- def end(self):
- self.te = time.time()
- if thread.get_ident() == mainThreadID:
- myLOG('bench THMain ' , self.__repr__())
- else:
- myLOG('bench TH%-6s' % thread.get_ident(),self.__repr__())
-
-
- def __repr__(self):
- return "%-60s: %8.3f sec" % (self.name,self.te-self.ts)
-
- def __str__(self):
- return self.__repr__()
-
-
class testZODB:
""" some wrapper stuff around ZODB """
@@ -301,9 +222,10 @@
class testSearches(dispatcher.Dispatcher,TestCase):
""" test searches """
- def __init__(self,func):
+ def __init__(self,func,args=(),kw={}):
- TestCase.__init__(self,func)
+
+ TestCase.__init__(self,func,args,kw)
dispatcher.Dispatcher.__init__(self)
self.init_phase = 0
@@ -337,7 +259,15 @@
self.zodb = self.catalog = None
def log_zodb_size(self,s):
- self.logn("Size of ZODB (data/work/Data.fs) %s test : %d" % (s,os.stat("data/work/Data.fs")[6]) )
+ self.logn("Size of ZODB (data/work/Data.fs) %s test : %s" % (s,self.size2size(os.stat("data/work/Data.fs")[6])) )
+
+
+ def size2size(self,n):
+ import math
+ if n <1024.0: return "%8.3lf Bytes" % n
+ if n <1024.0*1024.0: return "%8.3lf KB" % (1.0*n/1024.0)
+ if n <1024.0*1024.0*1024.0: return "%8.3lf MB" % (1.0*n/1024.0/1024.0)
+
#############################################################
@@ -345,15 +275,11 @@
#############################################################
- def testFulltextIndex(self,*args):
+ def testFulltextIndex(self,args,kw):
""" benchmark FulltextIndex """
- self.dispatcher('funcFulltextIndex' , ('funcFulltextIndex',1 , () , {} ) )
+ self.dispatcher('funcFulltextIndex' , ('funcFulltextIndex', kw["numThreads"] , () , {} ) )
- def testFulltextIndexTH(self,*args):
- """ thread benchmark FulltextIndex """
- self.dispatcher('funcFulltextIndex' , ('funcFulltextIndex',numThreads , () , {} ) )
-
def funcFulltextIndex(self,*args):
""" benchmark FulltextIndex """
@@ -371,13 +297,9 @@
# Field index test
#############################################################
- def testFieldIndex(self,*args):
+ def testFieldIndex(self,args,kw):
""" benchmark field index"""
- self.dispatcher('funcFieldIndex' , ('funcFieldIndex',1 , () , {} ) )
-
- def testFieldIndexTH(self,*args):
- """ thread benchmark field index"""
- self.dispatcher('funcFieldIndex' , ('funcFieldIndex',numThreads , () , {} ) )
+ self.dispatcher('funcFieldIndex' , ('funcFieldIndex',kw["numThreads"] , () , {} ) )
def funcFieldIndex(self,*args):
@@ -399,15 +321,11 @@
# Keyword index test
#############################################################
- def testKeywordIndex(self,*args):
+ def testKeywordIndex(self,args,kw):
""" benchmark Keyword index"""
- self.dispatcher('funcKeywordIndex' , ('funcKeywordIndex',1 , () , {} ) )
+ self.dispatcher('funcKeywordIndex' , ('funcKeywordIndex', kw["numThreads"] , () , {} ) )
- def testKeywordIndexTH(self,*args):
- """ thread benchmark Keyword index"""
- self.dispatcher('funcKeywordIndex' , ('funcKeywordIndex',numThreads, () , {} ) )
-
def funcKeywordIndex(self,*args):
""" benchmark KeywordIndex """
@@ -425,13 +343,9 @@
# Field range index test
#############################################################
- def testFieldRangeIndex(self,*args):
+ def testFieldRangeIndex(self,args,kw):
""" benchmark field range index"""
- self.dispatcher('funcFieldRangeIndex' , ('funcFieldRangeIndex',1 , () , {} ) )
-
- def testFieldRangeIndexTH(self,*args):
- """ thread benchmark field range index"""
- self.dispatcher('funcFieldRangeIndex' , ('funcFieldRangeIndex',numThreads, () , {} ) )
+ self.dispatcher('funcFieldRangeIndex' , ('funcFieldRangeIndex', kw["numThreads"] , () , {} ) )
def funcFieldRangeIndex(self,*args):
@@ -465,13 +379,9 @@
# Keyword + range index test
#############################################################
- def testKeywordRangeIndex(self,*args):
+ def testKeywordRangeIndex(self,args,kw):
""" benchmark Keyword range index"""
- self.dispatcher('funcKeywordRangeIndex' , ('funcKeywordRangeIndex',1 , () , {} ) )
-
- def testKeywordRangeIndexTH(self,*args):
- """ thread benchmark keyword range index"""
- self.dispatcher('funcKeywordRangeIndex' , ('funcKeywordRangeIndex',numThreads , () , {} ) )
+ self.dispatcher('funcKeywordRangeIndex' , ('funcKeywordRangeIndex', kw["numThreads"] , () , {} ) )
def funcKeywordRangeIndex(self,*args):
@@ -501,7 +411,6 @@
conflicts = 0
cat,msg_ids ,keywords= self.get_catalog()
- T = Timer('testUpdates of objects (100 iterations)')
for i in range(updateIterations):
@@ -521,7 +430,6 @@
except:
conflicts = conflicts + 1
- T.end()
self.conflicts[thread.get_ident()] = conflicts
self.threads[thread.get_ident()] = 1
@@ -532,7 +440,7 @@
# Test full reindexing
#############################################################
- def testReindexing(self,*args):
+ def testReindexing(self,args,kw):
""" test reindexing of existing data """
self.dispatcher("testReindexing" , ("funcReindexing",1 , (mbox,1000) , {} ))
@@ -548,8 +456,6 @@
mb = mailbox.UnixMailbox(open(mbox,"r"))
i = 0
- print numfiles
-
msg = mb.next()
while msg and i<numfiles:
@@ -564,8 +470,7 @@
msg = mb.next()
i = i+1
-# if i%100==0: print i
- print i
+ if i%100==0: print i
env = self.th_teardown(env,conflicts=conflicts)
@@ -574,10 +479,9 @@
# Test full reindexing
#############################################################
- def testIncrementalIndexing(self,*args):
+ def testIncrementalIndexing(self,args,kw):
""" testing incremental indexing """
-
- self.dispatcher("testIncrementalIndexing" , ("funcReindexing",1, (mbox2,100) , {}))
+ self.dispatcher("testIncrementalIndexing" , ("funcReindexing",1, (mbox2,1000) , {}))
def get_catalog(self):
@@ -645,16 +549,16 @@
if '-b' in optsLst:
basic_tests = [
- testSearches("testFulltextIndex"),
- testSearches("testFulltextIndexTH"),
- testSearches("testFieldIndex"),
- testSearches("testFieldIndexTH"),
- testSearches("testFieldRangeIndex"),
- testSearches("testFieldRangeIndexTH"),
- testSearches("testKeywordIndex"),
- testSearches("testKeywordIndexTH"),
- testSearches("testKeywordRangeIndex"),
- testSearches("testKeywordRangeIndexTH")
+ testSearches("testFulltextIndex",(),{"numThreads": 1}),
+ testSearches("testFulltextIndex",(),{"numThreads": 4}),
+ testSearches("testFieldIndex",(),{"numThreads": 1}),
+ testSearches("testFieldIndex",(),{"numThreads": 4}),
+ testSearches("testFieldRangeIndex",(),{"numThreads": 1}),
+ testSearches("testFieldRangeIndex",(),{"numThreads": 4}),
+ testSearches("testKeywordIndex",(),{"numThreads": 1}),
+ testSearches("testKeywordIndex",(),{"numThreads": 4}),
+ testSearches("testKeywordRangeIndex",(),{"numThreads": 1}),
+ testSearches("testKeywordRangeIndex",(),{"numThreads": 4})
]
testsuite1 = TestSuite()
@@ -666,9 +570,9 @@
if '-a' in optsLst:
basic_tests = [
- testSearches("testReindexing"),
+ testSearches("testReindexing",(),{"numThreads" : 1}),
+ testSearches("testIncrementalIndexing",(),{"numThreads" : 1})
# testSearches("testUpdates"),
-# testSearches("testIncrementalIndexing")
]
testsuite1 = TestSuite()