[Zodb-checkins] CVS: Zope/lib/python/ZEO - ClientCache.py:1.43.16.1 stats.py:1.20.16.1

Shane Hathaway shane at zope.com
Sat Jun 7 14:50:09 EDT 2003


Update of /cvs-repository/Zope/lib/python/ZEO
In directory cvs.zope.org:/tmp/cvs-serv3254

Modified Files:
      Tag: shane-oid-length-branch
	ClientCache.py stats.py 
Log Message:
Changed the client cache to allow arbitrary length OIDs.


=== Zope/lib/python/ZEO/ClientCache.py 1.43 => 1.43.16.1 ===
--- Zope/lib/python/ZEO/ClientCache.py:1.43	Tue Jan  7 17:39:53 2003
+++ Zope/lib/python/ZEO/ClientCache.py	Sat Jun  7 13:49:38 2003
@@ -44,7 +44,9 @@
 
   offset in record: name -- description
 
-  0: oid -- 8-byte object id
+  0: oidlen -- 2-byte unsigned object id length
+
+  2: reserved (6 bytes)
 
   8: status -- 1-byte status 'v': valid, 'n': non-version valid, 'i': invalid
                ('n' means only the non-version data in the record is valid)
@@ -57,23 +59,25 @@
 
   19: serial -- 8-byte non-version serial (timestamp)
 
-  27: data -- non-version data
+  27: oid -- object id
+
+  27+oidlen: data -- non-version data
 
-  27+dlen: version -- Version string (if vlen > 0)
+  27+oidlen+dlen: version -- Version string (if vlen > 0)
 
-  27+dlen+vlen: vdlen -- 4-byte length of version data (if vlen > 0)
+  27+oidlen+dlen+vlen: vdlen -- 4-byte length of version data (if vlen > 0)
 
-  31+dlen+vlen: vdata -- version data (if vlen > 0)
+  31+oidlen+dlen+vlen: vdata -- version data (if vlen > 0)
 
-  31+dlen+vlen+vdlen: vserial -- 8-byte version serial (timestamp)
+  31+oidlen+dlen+vlen+vdlen: vserial -- 8-byte version serial (timestamp)
                                  (if vlen > 0)
 
-  27+dlen (if vlen == 0) **or**
-  39+dlen+vlen+vdlen: tlen -- 4-byte (unsigned) record length (for
-                              redundancy and backward traversal)
+  27+oidlen+dlen (if vlen == 0) **or**
+  39+oidlen+dlen+vlen+vdlen: tlen -- 4-byte (unsigned) record length (for
+                                     redundancy and backward traversal)
 
-  31+dlen (if vlen == 0) **or**
-  43+dlen+vlen+vdlen: -- total record length (equal to tlen)
+  31+oidlen+dlen (if vlen == 0) **or**
+  43+oidlen+dlen+vlen+vdlen: -- total record length (equal to tlen)
 
 There is a cache size limit.
 
@@ -111,12 +115,12 @@
 from struct import pack, unpack
 from thread import allocate_lock
 
-from ZODB.utils import U64
+from ZODB.utils import oid_repr
 
 import zLOG
 from ZEO.ICache import ICache
 
-magic = 'ZEC1'
+magic = 'ZEC2'
 headersize = 12
 
 MB = 1024**2
@@ -293,15 +297,17 @@
             f.seek(ap)
             h = f.read(27)
             if len(h) != 27:
-                self.log("invalidate: short record for oid %16x "
+                self.log("invalidate: short record for oid %s "
                          "at position %d in cache file %d"
-                         % (U64(oid), ap, p < 0))
+                         % (oid_repr(oid), ap, p < 0))
                 del self._index[oid]
                 return None
-            if h[:8] != oid:
-                self.log("invalidate: oid mismatch: expected %16x read %16x "
+            oidlen = unpack(">H", h[:2])[0]
+            rec_oid = f.read(oidlen)
+            if rec_oid != oid:
+                self.log("invalidate: oid mismatch: expected %s read %s "
                          "at position %d in cache file %d"
-                         % (U64(oid), U64(h[:8]), ap, p < 0))
+                         % (oid_repr(oid), oid_repr(rec_oid), ap, p < 0))
                 del self._index[oid]
                 return None
             f.seek(ap+8) # Switch from reading to writing
@@ -329,14 +335,16 @@
             read = f.read
             seek(ap)
             h = read(27)
-            if len(h)==27 and h[8] in 'nv' and h[:8]==oid:
+            oidlen = unpack(">H", h[:2])[0]
+            rec_oid = read(oidlen)
+            if len(h)==27 and h[8] in 'nv' and rec_oid == oid:
                 tlen, vlen, dlen = unpack(">iHi", h[9:19])
             else:
                 tlen = -1
             if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen:
-                self.log("load: bad record for oid %16x "
+                self.log("load: bad record for oid %s "
                          "at position %d in cache file %d"
-                         % (U64(oid), ap, p < 0))
+                         % (oid_repr(oid), ap, p < 0))
                 del self._index[oid]
                 return None
 
@@ -355,7 +363,8 @@
                     data = read(dlen)
                     self._trace(0x2A, oid, version, h[19:], dlen)
                     if (p < 0) != self._current:
-                        self._copytocurrent(ap, tlen, dlen, vlen, h, data)
+                        self._copytocurrent(ap, oidlen, tlen, dlen, vlen, h,
+                                            oid, data)
                     return data, h[19:]
                 else:
                     self._trace(0x26, oid, version)
@@ -367,12 +376,12 @@
             v = vheader[:-4]
             if version != v:
                 if dlen:
-                    seek(ap+27)
+                    seek(ap+27+oidlen)
                     data = read(dlen)
                     self._trace(0x2C, oid, version, h[19:], dlen)
                     if (p < 0) != self._current:
-                        self._copytocurrent(ap, tlen, dlen, vlen, h,
-                                            data, vheader)
+                        self._copytocurrent(ap, oidlen, tlen, dlen, vlen, h,
+                                            oid, data, vheader)
                     return data, h[19:]
                 else:
                     self._trace(0x28, oid, version)
@@ -383,13 +392,13 @@
             vserial = read(8)
             self._trace(0x2E, oid, version, vserial, vdlen)
             if (p < 0) != self._current:
-                self._copytocurrent(ap, tlen, dlen, vlen, h,
-                                    None, vheader, vdata, vserial)
+                self._copytocurrent(ap, oidlen, tlen, dlen, vlen, h,
+                                    oid, None, vheader, vdata, vserial)
             return vdata, vserial
         finally:
             self._release()
 
-    def _copytocurrent(self, pos, tlen, dlen, vlen, header,
+    def _copytocurrent(self, pos, oidlen, tlen, dlen, vlen, header, oid,
                        data=None, vheader=None, vdata=None, vserial=None):
         """Copy a cache hit from the non-current file to the current file.
 
@@ -403,26 +412,27 @@
         if header[8] == 'n':
             # Rewrite the header to drop the version data.
             # This shortens the record.
-            tlen = 31 + dlen
+            tlen = 31 + oidlen + dlen
             vlen = 0
-            # (oid:8, status:1, tlen:4, vlen:2, dlen:4, serial:8)
+            # (oidlen:2, reserved:6, status:1, tlen:4,
+            #  vlen:2, dlen:4, serial:8)
             header = header[:9] + pack(">IHI", tlen, vlen, dlen) + header[-8:]
         else:
             assert header[8] == 'v'
         f = self._f[not self._current]
         if data is None:
-            f.seek(pos+27)
+            f.seek(pos+27+oidlen)
             data = f.read(dlen)
             if len(data) != dlen:
                 return
-        l = [header, data]
+        l = [header, oid, data]
         if vlen:
             assert vheader is not None
             l.append(vheader)
             assert (vdata is None) == (vserial is None)
             if vdata is None:
                 vdlen = unpack(">I", vheader[-4:])[0]
-                f.seek(pos+27+dlen+vlen+4)
+                f.seek(pos+27+oidlen+dlen+vlen+4)
                 vdata = f.read(vdlen)
                 if len(vdata) != vdlen:
                     return
@@ -438,13 +448,12 @@
         g.seek(self._pos)
         g.writelines(l)
         assert g.tell() == self._pos + tlen
-        oid = header[:8]
         if self._current:
             self._index[oid] = - self._pos
         else:
             self._index[oid] = self._pos
         self._pos += tlen
-        self._trace(0x6A, header[:8], vlen and vheader[:-4] or '',
+        self._trace(0x6A, oid, vlen and vheader[:-4] or '',
                     vlen and vserial or header[-8:], dlen)
 
     def update(self, oid, serial, version, data):
@@ -462,7 +471,9 @@
                 read = f.read
                 seek(ap)
                 h = read(27)
-                if len(h)==27 and h[8] in 'nv' and h[:8]==oid:
+                oidlen = unpack(">H", h[:2])[0]
+                rec_oid = read(oidlen)
+                if len(h) == 27 and h[8] in 'nv' and rec_oid == oid:
                     tlen, vlen, dlen = unpack(">iHi", h[9:19])
                 else:
                     return self._store(oid, '', '', version, data, serial)
@@ -500,14 +511,16 @@
             read = f.read
             seek(ap)
             h = read(27)
-            if len(h)==27 and h[8] in 'nv' and h[:8]==oid:
+            oidlen = unpack(">H", h[:2])[0]
+            rec_oid = read(oidlen)
+            if len(h) == 27 and h[8] in 'nv' and rec_oid == oid:
                 tlen, vlen, dlen = unpack(">iHi", h[9:19])
             else:
                 tlen = -1
             if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen:
-                self.log("modifiedInVersion: bad record for oid %16x "
+                self.log("modifiedInVersion: bad record for oid %s "
                          "at position %d in cache file %d"
-                         % (U64(oid), ap, p < 0))
+                         % (oid_repr(oid), ap, p < 0))
                 del self._index[oid]
                 return None
 
@@ -579,7 +592,7 @@
         if not s:
             p = ''
             s = '\0\0\0\0\0\0\0\0'
-        tlen = 31 + len(p)
+        tlen = 31 + len(oid) + len(p)
         if version:
             tlen = tlen + len(version) + 12 + len(pv)
             vlen = len(version)
@@ -588,7 +601,8 @@
 
         stlen = pack(">I", tlen)
         # accumulate various data to write into a list
-        l = [oid, 'v', stlen, pack(">HI", vlen, len(p)), s]
+        l = [pack(">H6x", len(oid)), 'v', stlen,
+             pack(">HI", vlen, len(p)), s, oid]
         if p:
             l.append(p)
         if version:
@@ -641,11 +655,11 @@
         if version:
             code |= 0x80
         self._tracefile.write(
-            struct_pack(">ii8s8s",
+            struct_pack(">iiH8s",
                         time_time(),
                         (dlen+255) & 0x7fffff00 | code | self._current,
-                        oid,
-                        serial))
+                        len(oid),
+                        serial) + oid)
 
     def read_index(self, serial, fileindex):
         index = self._index
@@ -672,7 +686,8 @@
                 self.rilog("invalid header data", pos, fileindex)
                 break
 
-            oid = h[:8]
+            oidlen = unpack(">H", h[:2])[0]
+            oid = read(oidlen)
 
             if h[8] == 'v' and vlen:
                 seek(dlen+vlen, 1)
@@ -681,7 +696,7 @@
                     self.rilog("truncated record", pos, fileindex)
                     break
                 vdlen = unpack(">i", vdlen)[0]
-                if vlen+dlen+43+vdlen != tlen:
+                if vlen + oidlen + dlen + 43 + vdlen != tlen:
                     self.rilog("inconsistent lengths", pos, fileindex)
                     break
                 seek(vdlen, 1)
@@ -691,7 +706,7 @@
                     break
             else:
                 if h[8] in 'vn' and vlen == 0:
-                    if dlen+31 != tlen:
+                    if oidlen + dlen + 31 != tlen:
                         self.rilog("inconsistent nv lengths", pos, fileindex)
                     seek(dlen, 1)
                     if read(4) != h[9:13]:


=== Zope/lib/python/ZEO/stats.py 1.20 => 1.20.16.1 ===
--- Zope/lib/python/ZEO/stats.py:1.20	Fri Jan  3 17:07:38 2003
+++ Zope/lib/python/ZEO/stats.py	Sat Jun  7 13:49:38 2003
@@ -35,8 +35,9 @@
 0       4     timestamp (seconds since 1/1/1970)
 4       3     data size, in 256-byte increments, rounded up
 7       1     code (see below)
-8       8     object id
-16      8     serial number
+8       2     object id length
+10      8     serial number
+18  variable  object id
 
 The code at offset 7 packs three fields:
 
@@ -56,6 +57,7 @@
 import time
 import getopt
 import struct
+from types import StringType
 
 def usage(msg):
     print >>sys.stderr, msg
@@ -155,12 +157,16 @@
                 if not quiet:
                     print "Skipping 8 bytes at offset", offset-8
                 continue
-            r = f_read(16)
-            if len(r) < 16:
+            r = f_read(10)
+            if len(r) < 10:
                 break
-            offset += 16
+            offset += 10
             records += 1
-            oid, serial = struct_unpack(">8s8s", r)
+            oidlen, serial = struct_unpack(">H8s", r)
+            oid = f_read(oidlen)
+            if len(oid) != oidlen:
+                break
+            offset += oidlen
             if t0 is None:
                 t0 = ts
                 thisinterval = t0 / interval
@@ -197,11 +203,11 @@
                     bysizew[dlen] = d = bysizew.get(dlen) or {}
                     d[oid] = d.get(oid, 0) + 1
             if verbose:
-                print "%s %d %02x %016x %016x %1s %s" % (
+                print "%s %d %02x %s %16x %1s %s" % (
                     time.ctime(ts)[4:-5],
                     current,
                     code,
-                    U64(oid),
+                    oid_repr(oid),
                     U64(serial),
                     version,
                     dlen and str(dlen) or "")
@@ -345,6 +351,12 @@
 def U64(s):
     h, v = struct.unpack(">II", s)
     return (long(h) << 32) + v
+
+def oid_repr(oid):
+    if isinstance(oid, StringType) and len(oid) == 8:
+        return '%16x' % U64(oid)
+    else:
+        return repr(oid)
 
 def addcommas(n):
     sign, s = '', str(n)




More information about the Zodb-checkins mailing list