[Zodb-checkins] CVS: ZODB3/ZEO - cache.py:1.1.2.4

[Jeremy Hylton]

Update of /cvs-repository/ZODB3/ZEO
In directory cvs.zope.org:/tmp/cvs-serv5297/ZEO

Modified Files:
      Tag: ZODB3-mvcc-2-branch
	cache.py 
Log Message:
Improvements to serialization code.

Store the last tid (self.tid) in the file.
Handle reading non-current data.
Implement contents() using a different strategy.
Fix Q -> 8s.
Change the DLL to return None instead of False for objects that aren't
in the cache.


=== ZODB3/ZEO/cache.py 1.1.2.3 => 1.1.2.4 ===
--- ZODB3/ZEO/cache.py:1.1.2.3	Mon Nov 10 17:42:54 2003
+++ ZODB3/ZEO/cache.py	Tue Nov 11 16:45:49 2003
@@ -14,6 +14,7 @@
 """Example client cache that stores multiple revisions of an object."""
 
 import bisect
+import os
 import struct
 
 from sets import Set
@@ -100,31 +101,43 @@
         self.dll = Cache(size or 10**6)
 
     def open(self):
-        self._read()
+        if os.path.exists(self.path):
+            self._read()
 
     def _read(self):
-        f = open(self.path, "rb")
+        f = open(self.path, "a+b")
+        tid = f.read(8)
+        if tid == "\0" * 8:
+            self.tid = None
+        else:
+            self.tid = tid
         while 1:
             o = Object.fromFile(f, self)
             if o is None:
                 break
+            oid = o.key[0]
             if o.version:
-                self.version[o.oid] = o.start_tid
+                self.version[oid] = o.start_tid
             elif o.end_tid is None:
-                self.current[o.oid] = o.start_tid
+                self.current[oid] = o.start_tid
             else:
-                # XXX recreate the internal cache data structures
-                print "non-current cache data"
+                L = self.noncurrent.setdefault(oid, [])
+                bisect.insort_left(L, (o.start_tid, o.end_tid))
+            self.dll.add(o)
 
     def close(self):
         self._write()
 
     def _write(self):
+        if self.path is None:
+            return
         # XXX The cache needs a minimal header.
         # magic cookie, format version no, configured size of cache
+        # last tid
         f = open(self.path, "wb")
+        f.write(self.tid or "\0" * 8)
         for o in self.dll:
-            o.write(f)
+            o.serialize(f)
         f.close()
 
     ##
@@ -160,6 +173,8 @@
         if tid is None:
             return None
         o = self.dll.access((oid, tid))
+        if o is None:
+            return None
         return o.data, o.serialno
 
     ##
@@ -250,31 +265,31 @@
         cur_tid = self.current.pop(oid)
         # XXX Want to fetch object without marking it as accessed
         o = self.dll.access((oid, cur_tid))
+        if o is None:
+            return None
         o.end_tid = tid
         L = self.noncurrent.setdefault(oid, [])
         bisect.insort_left(L, (cur_tid, tid))
 
     ##
-    # An iterator yielding the current contents of the cache.
-    # @defreturn iterator
-    # @return oid, version, serial number triples
-
-    def contents(self):
-        # XXX
-        for oid, (data, serial) in self.current.items():
-            yield oid, "", serial
-        for oid, (version, data, serial, start_tid) in self.version.items():
-            yield oid, version, serial
-
-    ##
     # Return the number of object revisions in the cache.
 
     def __len__(self):
         n = len(self.current) + len(self.version)
         if self.noncurrent:
-            n += reduce(int.__add__, map(len, self.noncurrent))
+            n += sum(map(len, self.noncurrent))
         return n
 
+    ##
+    # Generates over, version, serial triples for all objects in the
+    # cache.  This generator is used by cache verification.
+
+    def contents(self):
+        # XXX May need to materialize list instead of iterating,
+        # depends on whether the caller may change the cache.
+        for o in self.dll:
+            yield o.key[0], o.version, o.serialno
+
     def _evicted(self, o):
         # Called by Object o to signal its eviction
         oid, tid = o.key
@@ -376,22 +391,29 @@
         DLLNode.unlink(self)
         self.cache._evicted(self)
 
+    # The serialization format uses an end tid of "\0" * 8, the least
+    # 8-byte string, to represent None.  It isn't possible for an
+    # end_tid to be 0, because it must always be strictly greater
+    # than the start_tid.
+
     def serialize(self, f):
         # Write standard form of Object to file, f.
-        s = struct.pack(">QQQQhi", self.key[0], self.serialno,
-                        self.start_tid, self.end_tid, len(self.version),
-                        len(self.data))
+        s = struct.pack(">8s8s8s8shi", self.key[0], self.serialno,
+                        self.start_tid, self.end_tid or "\0" * 8,
+                        len(self.version), len(self.data))
         f.write(s)
         f.write(self.version)
         f.write(self.data)
-        f.write(struct.pack(">Q", s))
+        f.write(struct.pack(">8s", s))
 
     def fromFile(cls, f, cache):
-        fmt = ">QQQQhi"
+        fmt = ">8s8s8s8shi"
         s = f.read(struct.calcsize(fmt))
         if not s:
             return None
         oid, serialno, start_tid, end_tid, vlen, dlen = struct.unpack(fmt, s)
+        if end_tid == "\0" * 8:
+            end_tid = None
         version = f.read(vlen)
         if vlen != len(version):
             raise ValueError("corrupted record, version")
@@ -399,7 +421,7 @@
         if dlen != len(data):
             raise ValueError("corrupted record, data")
         s = f.read(8)
-        if struct.pack(">Q", s) != oid:
+        if struct.pack(">8s", s) != oid:
             raise ValueError("corrupted record, oid")
         return cls((oid, start_tid), version, serialno, data,
                    start_tid, end_tid, cache)
@@ -535,7 +557,7 @@
         self._tick()
         obj = self.key2object.get(oid)
         if obj is None:
-            return False # XXX None?
+            return None
         self._change_worth(obj, obj.worth | 0x80)
         return obj