[Zodb-checkins] CVS: ZODB3/ZEO - simul.py:1.12.8.2.18.3

Jeremy Hylton cvs-admin at zope.org
Tue Dec 2 22:45:05 EST 2003


Update of /cvs-repository/ZODB3/ZEO
In directory cvs.zope.org:/tmp/cvs-serv1324

Modified Files:
      Tag: Zope-2_6-branch
	simul.py 
Log Message:
Add circular cache simulation.
Extend reporting code to support multiple simulator-specific
statistics.


=== ZODB3/ZEO/simul.py 1.12.8.2.18.2 => 1.12.8.2.18.3 ===
--- ZODB3/ZEO/simul.py:1.12.8.2.18.2	Wed Nov 26 16:44:39 2003
+++ ZODB3/ZEO/simul.py	Tue Dec  2 22:45:00 2003
@@ -43,8 +43,9 @@
     MB = 1000*1000
     cachelimit = 20*MB
     simclass = ZEOCacheSimulation
+    omicron = None
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "bflyz2s:")
+        opts, args = getopt.getopt(sys.argv[1:], "bflyz2cs:o:")
     except getopt.error, msg:
         usage(msg)
         return 2
@@ -63,11 +64,19 @@
             cachelimit = int(float(a)*MB)
         if o == '-2':
             simclass = TwoQSimluation
+        if o == '-c':
+            simclass = CircularCacheSimulation
+        if o == '-o':
+            omicron = float(a)
     if len(args) != 1:
         usage("exactly one file argument required")
         return 2
     filename = args[0]
 
+    if omicron is not None and simclass != CircularCacheSimulation:
+        usage("-o flag only useful with -c (CircularCacheSimulation)")
+        return 2
+
     # Open file
     if filename.endswith(".gz"):
         # Open gzipped file
@@ -93,7 +102,10 @@
             return 1
 
     # Create simulation object
-    sim = simclass(cachelimit)
+    if omicron is not None:
+        sim = simclass(cachelimit, omicron)
+    else:
+        sim = simclass(cachelimit)
 
     # Print output header
     sim.printheader()
@@ -154,6 +166,9 @@
         self.total_hits = 0 # Subclass must increment
         self.total_invals = 0
         self.total_writes = 0
+        if not hasattr(self, "extras"):
+            self.extras = (self.extraname,)
+        self.format = self.format + " %6s" * len(self.extras)
         # Reset per-run statistics and set up simulation data
         self.restart()
 
@@ -204,7 +219,7 @@
     def inval(self, oid):
         pass
 
-    format = "%12s %9s %8s %8s %6s %6s %6s %6s"
+    format = "%12s %9s %8s %8s %6s %6s %6s"
 
     # Subclass should override extraname to name known instance variables;
     # if extraname is 'foo', both self.foo and self.total_foo must exist:
@@ -213,34 +228,41 @@
     def printheader(self):
         print "%s, cache size %s bytes" % (self.__class__.__name__,
                                            addcommas(self.cachelimit))
-        print self.format % (
-            "START TIME", "DURATION", "LOADS", "HITS",
-            "INVALS", "WRITES", self.extraname.upper(), "HITRATE")
+        self.extraheader()
+        extranames = tuple([s.upper() for s in self.extras])
+        args = ("START TIME", "DURATION", "LOADS", "HITS",
+                "INVALS", "WRITES", "HITRATE") + extranames
+        print self.format % args
+
+    def extraheader(self):
+        pass
 
     nreports = 0
 
     def report(self):
         if self.loads:
             self.nreports += 1
-            print self.format % (
-                time.ctime(self.ts0)[4:-8],
-                duration(self.ts1 - self.ts0),
-                self.loads, self.hits, self.invals, self.writes,
-                getattr(self, self.extraname),
-                hitrate(self.loads, self.hits))
+            args = (time.ctime(self.ts0)[4:-8],
+                    duration(self.ts1 - self.ts0),
+                    self.loads, self.hits, self.invals, self.writes,
+                    hitrate(self.loads, self.hits))
+            args += tuple([getattr(self, name) for name in self.extras])
+            print self.format % args
 
     def finish(self):
         self.report()
         if self.nreports > 1:
-            print (self.format + " OVERALL") % (
+            args = (
                 time.ctime(self.epoch)[4:-8],
                 duration(self.ts1 - self.epoch),
                 self.total_loads,
                 self.total_hits,
                 self.total_invals,
                 self.total_writes,
-                getattr(self, "total_" + self.extraname),
                 hitrate(self.total_loads, self.total_hits))
+            args += tuple([getattr(self, "total_" + name)
+                           for name in self.extras])
+            print (self.format + " OVERALL") % args
 
 class ZEOCacheSimulation(Simulation):
 
@@ -442,11 +464,12 @@
     # object based, so, for example, it's hard to compute the number
     # of oids to store in a1out based on the size of a1in.
 
-    extraname = "evicts"
+    extras = "evicts", "hothits"
 
     def __init__(self, cachelimit, outlen=100000):
         Simulation.__init__(self, cachelimit)
         self.total_evicts = 0
+        self.total_hothits = 0
         self.a1out_limit = outlen
 
         # An LRU queue of hot objects
@@ -535,6 +558,8 @@
             if node.kind is am:
                 self.hits += 1
                 self.total_hits += 1
+                self.hothits += 1
+                self.total_hothits += 1
                 node.linkbefore(self.am)
             elif node.kind is a1in:
                 self.hits += 1
@@ -574,6 +599,7 @@
         Simulation.restart(self)
 
         self.evicts = 0
+        self.hothits = 0
 
         self.cache = {}
         self.am_limit = 3 * self.cachelimit / 4
@@ -594,6 +620,146 @@
         if next.kind != self.kind:
             self.kind = next.kind
         Node.linkbefore(self, next)
+
+class CircularCacheSimulation(Simulation):
+
+    # The cache is managed as a single file with a pointer that
+    # goes around the file, circularly, forever.  New objects
+    # are written at the current pointer, evicting whatever was
+    # there previously.
+
+    # For each cache hit, there is some distance between the current
+    # pointer offset and the offset of the cached data record.  The
+    # cache can be configured to copy objects to the current offset
+    # depending on how far away they are now.  The omicron parameter
+    # specifies a percentage 
+
+    extras = "evicts", "copies"
+
+    def __init__(self, cachelimit, omicron=0):
+        Simulation.__init__(self, cachelimit)
+        self.omicron = omicron
+        self.total_evicts = 0
+        self.total_copies = 0
+        # Current offset in file
+        self.offset = 0
+        # Map offset in file to tuple of size, oid
+        self.filemap = {0: (self.cachelimit, None)}
+        # Map oid to offset
+        self.cache = {}
+
+    def extraheader(self):
+        print "omicron = %s" % self.omicron
+
+    def restart(self):
+        Simulation.restart(self)
+        self.evicts = 0
+        self.copies = 0
+
+    def load(self, oid, size):
+        pos = self.cache.get(oid)
+        if pos is None:
+            self.add(oid, size)
+        else:
+            self.hits += 1
+            self.total_hits += 1
+            self.copy(oid, size, pos)
+
+    def check(self):
+        d = dict(self.filemap)
+        done = {}
+        while d:
+            pos, (size, oid) = d.popitem()
+            next = pos + size
+            assert next in d or next in done or next == self.cachelimit
+            done[pos] = pos
+        print "checked", self.offset
+
+    def dump(self):
+        L = list(self.filemap)
+        L.sort()
+        print "cache size", len(L)
+        next = None
+        for pos in L:
+            size, oid = self.filemap[pos]
+            if oid:
+                print pos, size, repr(oid), self.cache[oid]
+            else:
+                print pos, size, None
+            if next is not None:
+                assert next == pos
+            next = pos + size
+        print
+
+    def add(self, oid, size):
+        avail = self.makeroom(size)
+        assert oid not in self.cache
+        self.filemap[self.offset] = size, oid
+        self.cache[oid] = self.offset
+        self.offset += size
+        # All the space made available must be accounted for in filemap.
+        excess = avail - size
+        if excess:
+            self.filemap[self.offset] = excess, None
+
+    def makeroom(self, need):
+        if self.offset + need > self.cachelimit:
+            self.offset = 0
+        pos = self.offset
+        # Evict enough objects to make the necessary space available.
+        while need > 0:
+            try:
+                size, oid = self.filemap.pop(pos)
+            except:
+                raise
+                    
+            if oid is not None:
+                self.evicts += 1
+                self.total_evicts += 1
+                del self.cache[oid]
+            pos += size
+            need -= size
+        return pos - self.offset
+
+    def copy(self, oid, size, pos):
+        # Copy only if the distance is greater than omicron.
+        dist = self.offset - pos
+        if dist < 0:
+            dist += self.cachelimit
+        if dist < self.omicron * self.cachelimit:
+            self.copies += 1
+            self.total_copies += 1
+            self.filemap[pos] = size, None
+            del self.cache[oid]
+            self.add(oid, size)
+
+    def inval(self, oid):
+        pos = self.cache.get(oid)
+        if pos is None:
+            return
+        self.invals += 1
+        self.total_invals += 1
+        size, _oid = self.filemap[pos]
+        assert oid == _oid
+        self.filemap[pos] = size, None
+        del self.cache[oid]
+
+    def write(self, oid, size):
+        pos = self.cache.get(oid)
+        if pos is None:
+            return
+        oldsize, _oid = self.filemap[pos]
+        assert oid == _oid
+        if size == oldsize:
+            return
+        if size < oldsize:
+            excess = oldsize - size
+            self.filemap[pos] = size, oid
+            self.filemap[pos + size] = excess, None
+        else:
+            self.filemap[pos] = oldsize, None
+            del self.cache[oid]
+            self.add(oid, size)
 
 class BuddyCacheSimulation(LRUCacheSimulation):
 




More information about the Zodb-checkins mailing list