[Zodb-checkins] CVS: ZODB3/ZEO - simul.py:1.12.8.2.18.3
Jeremy Hylton
cvs-admin at zope.org
Tue Dec 2 22:45:05 EST 2003
Update of /cvs-repository/ZODB3/ZEO
In directory cvs.zope.org:/tmp/cvs-serv1324
Modified Files:
Tag: Zope-2_6-branch
simul.py
Log Message:
Add circular cache simulation.
Extend reporting code to support multiple simulator-specific
statistics.
=== ZODB3/ZEO/simul.py 1.12.8.2.18.2 => 1.12.8.2.18.3 ===
--- ZODB3/ZEO/simul.py:1.12.8.2.18.2 Wed Nov 26 16:44:39 2003
+++ ZODB3/ZEO/simul.py Tue Dec 2 22:45:00 2003
@@ -43,8 +43,9 @@
MB = 1000*1000
cachelimit = 20*MB
simclass = ZEOCacheSimulation
+ omicron = None
try:
- opts, args = getopt.getopt(sys.argv[1:], "bflyz2s:")
+ opts, args = getopt.getopt(sys.argv[1:], "bflyz2cs:o:")
except getopt.error, msg:
usage(msg)
return 2
@@ -63,11 +64,19 @@
cachelimit = int(float(a)*MB)
if o == '-2':
simclass = TwoQSimluation
+ if o == '-c':
+ simclass = CircularCacheSimulation
+ if o == '-o':
+ omicron = float(a)
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
+ if omicron is not None and simclass != CircularCacheSimulation:
+ usage("-o flag only useful with -c (CircularCacheSimulation)")
+ return 2
+
# Open file
if filename.endswith(".gz"):
# Open gzipped file
@@ -93,7 +102,10 @@
return 1
# Create simulation object
- sim = simclass(cachelimit)
+ if omicron is not None:
+ sim = simclass(cachelimit, omicron)
+ else:
+ sim = simclass(cachelimit)
# Print output header
sim.printheader()
@@ -154,6 +166,9 @@
self.total_hits = 0 # Subclass must increment
self.total_invals = 0
self.total_writes = 0
+ if not hasattr(self, "extras"):
+ self.extras = (self.extraname,)
+ self.format = self.format + " %6s" * len(self.extras)
# Reset per-run statistics and set up simulation data
self.restart()
@@ -204,7 +219,7 @@
def inval(self, oid):
pass
- format = "%12s %9s %8s %8s %6s %6s %6s %6s"
+ format = "%12s %9s %8s %8s %6s %6s %6s"
# Subclass should override extraname to name known instance variables;
# if extraname is 'foo', both self.foo and self.total_foo must exist:
@@ -213,34 +228,41 @@
def printheader(self):
print "%s, cache size %s bytes" % (self.__class__.__name__,
addcommas(self.cachelimit))
- print self.format % (
- "START TIME", "DURATION", "LOADS", "HITS",
- "INVALS", "WRITES", self.extraname.upper(), "HITRATE")
+ self.extraheader()
+ extranames = tuple([s.upper() for s in self.extras])
+ args = ("START TIME", "DURATION", "LOADS", "HITS",
+ "INVALS", "WRITES", "HITRATE") + extranames
+ print self.format % args
+
+ def extraheader(self):
+ pass
nreports = 0
def report(self):
if self.loads:
self.nreports += 1
- print self.format % (
- time.ctime(self.ts0)[4:-8],
- duration(self.ts1 - self.ts0),
- self.loads, self.hits, self.invals, self.writes,
- getattr(self, self.extraname),
- hitrate(self.loads, self.hits))
+ args = (time.ctime(self.ts0)[4:-8],
+ duration(self.ts1 - self.ts0),
+ self.loads, self.hits, self.invals, self.writes,
+ hitrate(self.loads, self.hits))
+ args += tuple([getattr(self, name) for name in self.extras])
+ print self.format % args
def finish(self):
self.report()
if self.nreports > 1:
- print (self.format + " OVERALL") % (
+ args = (
time.ctime(self.epoch)[4:-8],
duration(self.ts1 - self.epoch),
self.total_loads,
self.total_hits,
self.total_invals,
self.total_writes,
- getattr(self, "total_" + self.extraname),
hitrate(self.total_loads, self.total_hits))
+ args += tuple([getattr(self, "total_" + name)
+ for name in self.extras])
+ print (self.format + " OVERALL") % args
class ZEOCacheSimulation(Simulation):
@@ -442,11 +464,12 @@
# object based, so, for example, it's hard to compute the number
# of oids to store in a1out based on the size of a1in.
- extraname = "evicts"
+ extras = "evicts", "hothits"
def __init__(self, cachelimit, outlen=100000):
Simulation.__init__(self, cachelimit)
self.total_evicts = 0
+ self.total_hothits = 0
self.a1out_limit = outlen
# An LRU queue of hot objects
@@ -535,6 +558,8 @@
if node.kind is am:
self.hits += 1
self.total_hits += 1
+ self.hothits += 1
+ self.total_hothits += 1
node.linkbefore(self.am)
elif node.kind is a1in:
self.hits += 1
@@ -574,6 +599,7 @@
Simulation.restart(self)
self.evicts = 0
+ self.hothits = 0
self.cache = {}
self.am_limit = 3 * self.cachelimit / 4
@@ -594,6 +620,146 @@
if next.kind != self.kind:
self.kind = next.kind
Node.linkbefore(self, next)
+
+class CircularCacheSimulation(Simulation):
+
+ # The cache is managed as a single file with a pointer that
+ # goes around the file, circularly, forever. New objects
+ # are written at the current pointer, evicting whatever was
+ # there previously.
+
+ # For each cache hit, there is some distance between the current
+ # pointer offset and the offset of the cached data record. The
+ # cache can be configured to copy objects to the current offset
+ # depending on how far away they are now. The omicron parameter
+ # specifies a percentage
+
+ extras = "evicts", "copies"
+
+ def __init__(self, cachelimit, omicron=0):
+ Simulation.__init__(self, cachelimit)
+ self.omicron = omicron
+ self.total_evicts = 0
+ self.total_copies = 0
+ # Current offset in file
+ self.offset = 0
+ # Map offset in file to tuple of size, oid
+ self.filemap = {0: (self.cachelimit, None)}
+ # Map oid to offset
+ self.cache = {}
+
+ def extraheader(self):
+ print "omicron = %s" % self.omicron
+
+ def restart(self):
+ Simulation.restart(self)
+ self.evicts = 0
+ self.copies = 0
+
+ def load(self, oid, size):
+ pos = self.cache.get(oid)
+ if pos is None:
+ self.add(oid, size)
+ else:
+ self.hits += 1
+ self.total_hits += 1
+ self.copy(oid, size, pos)
+
+ def check(self):
+ d = dict(self.filemap)
+ done = {}
+ while d:
+ pos, (size, oid) = d.popitem()
+ next = pos + size
+ assert next in d or next in done or next == self.cachelimit
+ done[pos] = pos
+ print "checked", self.offset
+
+ def dump(self):
+ L = list(self.filemap)
+ L.sort()
+ print "cache size", len(L)
+ next = None
+ for pos in L:
+ size, oid = self.filemap[pos]
+ if oid:
+ print pos, size, repr(oid), self.cache[oid]
+ else:
+ print pos, size, None
+ if next is not None:
+ assert next == pos
+ next = pos + size
+ print
+
+ def add(self, oid, size):
+ avail = self.makeroom(size)
+ assert oid not in self.cache
+ self.filemap[self.offset] = size, oid
+ self.cache[oid] = self.offset
+ self.offset += size
+ # All the space made available must be accounted for in filemap.
+ excess = avail - size
+ if excess:
+ self.filemap[self.offset] = excess, None
+
+ def makeroom(self, need):
+ if self.offset + need > self.cachelimit:
+ self.offset = 0
+ pos = self.offset
+ # Evict enough objects to make the necessary space available.
+ while need > 0:
+ try:
+ size, oid = self.filemap.pop(pos)
+ except:
+ raise
+
+ if oid is not None:
+ self.evicts += 1
+ self.total_evicts += 1
+ del self.cache[oid]
+ pos += size
+ need -= size
+ return pos - self.offset
+
+ def copy(self, oid, size, pos):
+ # Copy only if the distance is greater than omicron.
+ dist = self.offset - pos
+ if dist < 0:
+ dist += self.cachelimit
+ if dist < self.omicron * self.cachelimit:
+ self.copies += 1
+ self.total_copies += 1
+ self.filemap[pos] = size, None
+ del self.cache[oid]
+ self.add(oid, size)
+
+ def inval(self, oid):
+ pos = self.cache.get(oid)
+ if pos is None:
+ return
+ self.invals += 1
+ self.total_invals += 1
+ size, _oid = self.filemap[pos]
+ assert oid == _oid
+ self.filemap[pos] = size, None
+ del self.cache[oid]
+
+ def write(self, oid, size):
+ pos = self.cache.get(oid)
+ if pos is None:
+ return
+ oldsize, _oid = self.filemap[pos]
+ assert oid == _oid
+ if size == oldsize:
+ return
+ if size < oldsize:
+ excess = oldsize - size
+ self.filemap[pos] = size, oid
+ self.filemap[pos + size] = excess, None
+ else:
+ self.filemap[pos] = oldsize, None
+ del self.cache[oid]
+ self.add(oid, size)
class BuddyCacheSimulation(LRUCacheSimulation):
More information about the Zodb-checkins
mailing list