[Zodb-checkins] CVS: ZODB4/ZODB - fsdump.py:1.10 TimeStamp.c:1.19 Serialize.py:1.10 POSException.py:1.19 FileStorage.py:1.112 BaseStorage.py:1.26

Jeremy Hylton jeremy@zope.com
Wed, 11 Dec 2002 18:40:51 -0500


Update of /cvs-repository/ZODB4/ZODB
In directory cvs.zope.org:/tmp/cvs-serv19253/ZODB

Modified Files:
	fsdump.py TimeStamp.c Serialize.py POSException.py 
	FileStorage.py BaseStorage.py 
Log Message:
Merge the ZODB4-Q-branch to the trunk.

The primary effect of this change is to use take advantage of the Q
struct format code to simplify FileStorage.  There were many places
where FileStorage used "8s" and then had to call u64() on the result
to convert from an 8-byte string to a Python long.  This was tedious
and added lots of extra noise to the FileStorage code.

Along the way, a lot of other refactoring happened.  Introduce a
FileStorageFormatter that handles most of the details of reading the
low-level FileStorage format.  It returns a simply Python object that
has attributes for each of the fields in a data header.  The formatter
class has been used in many places, but the changes aren't complete.
It should be used for the transaction header and for writing the data
header, but I don't know when I'll get to that.

The pack implementation was refactored a bit.  Most of the logic was
moved to a separate class FileStoragePacker, because the _file
attribute is a separate file object opened on the same underlying file
as the main FileStorage.

Since old-style undo is no longer supported, all the checks for 'u'
were removed and the _sane() method is no longer necessary.

The code passes all the tests, but there are a bunch of corner cases
in pack() that should be tested more thoroughly.  A bunch of branches
aren't executed during the pack tests.

A run of speed.py shows that the code is faster on the whole.  It's
quicker to get the Python long directly from struct than it is to get
an 8s and call another Python function to convert it.  For the speed
tests, the runs with small # of objects are slower (roughly 2x), but
the runs with large # of objects are faster (roughly 50%).


=== ZODB4/ZODB/fsdump.py 1.9 => 1.10 ===
--- ZODB4/ZODB/fsdump.py:1.9	Tue Dec  3 18:37:08 2002
+++ ZODB4/ZODB/fsdump.py	Wed Dec 11 18:40:50 2002
@@ -21,6 +21,9 @@
     # Return a nicely formatted string for a packaged 64-bit value
     return "%016x" % u64(p64)
 
+def dump(path, dest=None):
+    Dumper(path, dest).dump()
+
 class Dumper:
     """A very verbose dumper for debugging FileStorage problems."""
 
@@ -40,13 +43,13 @@
         h = self.file.read(TRANS_HDR_LEN)
         if not h:
             return False
-        tid, stlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
-        end = pos + u64(stlen)
+        tid, tlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+        end = pos + tlen
         print >> self.dest, "=" * 60
         print >> self.dest, "offset: %d" % pos
         print >> self.dest, "end pos: %d" % end
         print >> self.dest, "transaction id: %s" % fmt(tid)
-        print >> self.dest, "trec len: %d" % u64(stlen)
+        print >> self.dest, "trec len: %d" % tlen
         print >> self.dest, "status: %r" % status
         user = descr = extra = ""
         if ul:
@@ -60,22 +63,21 @@
         print >> self.dest, "len(extra): %d" % el
         while self.file.tell() < end:
             self.dump_data(pos)
-        stlen2 = self.file.read(8)
-        print >> self.dest, "redundant trec len: %d" % u64(stlen2)
+        tlen2 = u64(self.file.read(8))
+        print >> self.dest, "redundant trec len: %d" % tlen2
         return True
 
     def dump_data(self, tloc):
         pos = self.file.tell()
         h = self.file.read(DATA_HDR_LEN)
         assert len(h) == DATA_HDR_LEN
-        oid, revid, sprev, stloc, vlen, sdlen = struct.unpack(DATA_HDR, h)
-        dlen = u64(sdlen)
+        oid, revid, prev, tloc, vlen, dlen = struct.unpack(DATA_HDR, h)
         print >> self.dest, "-" * 60
         print >> self.dest, "offset: %d" % pos
         print >> self.dest, "oid: %s" % fmt(oid)
         print >> self.dest, "revid: %s" % fmt(revid)
-        print >> self.dest, "previous record offset: %d" % u64(sprev)
-        print >> self.dest, "transaction offset: %d" % u64(stloc)
+        print >> self.dest, "previous record offset: %d" % prev
+        print >> self.dest, "transaction offset: %d" % tloc
         if vlen:
             pnv = self.file.read(8)
             sprevdata = self.file.read(8)


=== ZODB4/ZODB/TimeStamp.c 1.18 => 1.19 ===
--- ZODB4/ZODB/TimeStamp.c:1.18	Mon Nov 25 16:52:27 2002
+++ ZODB4/ZODB/TimeStamp.c	Wed Dec 11 18:40:50 2002
@@ -105,7 +105,10 @@
 static int
 TimeStamp_compare(TimeStamp *v, TimeStamp *w)
 {
-    return memcmp(v->data, w->data, 8);
+    int cmp = memcmp(v->data, w->data, 8);
+    if (cmp < 0) return -1;
+    if (cmp > 0) return 1;
+    return 0;
 }
 
 static long


=== ZODB4/ZODB/Serialize.py 1.9 => 1.10 ===
--- ZODB4/ZODB/Serialize.py:1.9	Tue Dec  3 18:07:05 2002
+++ ZODB4/ZODB/Serialize.py	Wed Dec 11 18:40:50 2002
@@ -63,13 +63,14 @@
         mod = getattr(mod, part)
     return getattr(mod, name)
 
-def getClassMetadata(object=None, klass=None):
+def getClassMetadata(obj=None, klass=None):
     if klass is None:
-        klass = object.__class__
+        klass = obj.__class__
     module = klass.__module__
     classname = klass.__name__
-    if hasattr(object, "__getnewargs__"):
-        newargs = object.__getnewargs__()
+    # XXX what if obj is None and we were passed klass?
+    if hasattr(obj, "__getnewargs__"):
+        newargs = obj.__getnewargs__()
     else:
         newargs = None
     return module, classname, newargs
@@ -177,11 +178,11 @@
     def _new_object(self, module, classname, newargs=None):
         klass = getClass(module, classname)
         if newargs is None:
-            object = klass.__new__(klass)
+            obj = klass.__new__(klass)
         else:
-            object = klass.__new__(klass, *newargs)
+            obj = klass.__new__(klass, *newargs)
                 
-        return object
+        return obj
 
     def getClassName(self, pickle):
         unpickler = self._get_unpickler(pickle)
@@ -206,10 +207,10 @@
     def getObject(self, pickle):
         unpickler = self._get_unpickler(pickle)
         module, classname, newargs = unpickler.load()
-        object = self._new_object(module, classname, newargs)
+        obj = self._new_object(module, classname, newargs)
         state = unpickler.load()
-        object.__setstate__(state)
-        return object
+        obj.__setstate__(state)
+        return obj
 
 class SimpleObjectReader(BaseObjectReader):
     """Minimal reader for a single data record."""
@@ -231,23 +232,23 @@
             # Quick instance reference.  We know all we need to know
             # to create the instance w/o hitting the db, so go for it!
             oid, classmeta = oid
-            object = self._cache.get(oid)
-            if object is not None:
-                return object
+            obj = self._cache.get(oid)
+            if obj is not None:
+                return obj
 
-            object = self._new_object(*classmeta)
+            obj = self._new_object(*classmeta)
 
             # XXX should be done by connection
-            object._p_oid = oid
-            object._p_jar = self._conn
-            object._p_changed = None
+            obj._p_oid = oid
+            obj._p_jar = self._conn
+            obj._p_changed = None
             
-            self._cache[oid] = object
-            return object
+            self._cache[oid] = obj
+            return obj
 
-        object = self._cache.get(oid)
-        if object is not None:
-            return object
+        obj = self._cache.get(oid)
+        if obj is not None:
+            return obj
         return self._conn[oid]
 
 class CopyReference:


=== ZODB4/ZODB/POSException.py 1.18 => 1.19 ===
--- ZODB4/ZODB/POSException.py:1.18	Tue Dec  3 13:38:02 2002
+++ ZODB4/ZODB/POSException.py	Wed Dec 11 18:40:50 2002
@@ -152,6 +152,14 @@
 class VersionLockError(VersionError, TransactionError):
     """Can't modify an object that is modified in unsaved version."""
 
+    def __init__(self, oid, version):
+        self.oid = oid
+        self.version = version
+
+    def __str__(self):
+        return "%s locked in version %r" % (_fmt_oid(self.oid),
+                                            self.version)
+
 class UndoError(POSError):
     """An attempt was made to undo a non-undoable transaction."""
 


=== ZODB4/ZODB/FileStorage.py 1.111 => 1.112 === (1852/1952 lines abridged)
--- ZODB4/ZODB/FileStorage.py:1.111	Fri Dec  6 18:09:29 2002
+++ ZODB4/ZODB/FileStorage.py	Wed Dec 11 18:40:50 2002
@@ -139,28 +139,23 @@
 from ZODB.TimeStamp import TimeStamp, newTimeStamp, timeStampFromTime
 from ZODB.lock_file import lock_file
 from ZODB.utils import p64, u64, cp, z64
-
-try:
-    from ZODB.fsIndex import fsIndex
-except ImportError:
-    def fsIndex():
-        return {}
+from ZODB.fsIndex import fsIndex
 
 from zLOG import LOG, BLATHER, WARNING, ERROR, PANIC
 
 t32 = 1L << 32
 # the struct formats for the headers
-TRANS_HDR = ">8s8scHHH"
-DATA_HDR = ">8s8s8s8sH8s"
+TRANS_HDR = ">8sQcHHH"
+DATA_HDR = ">8s8sQQHQ"
 # constants to support various header sizes
 TRANS_HDR_LEN = 23
 DATA_HDR_LEN = 42
 DATA_VERSION_HDR_LEN = 58
-
-packed_version='FS21'
 assert struct.calcsize(TRANS_HDR) == TRANS_HDR_LEN
 assert struct.calcsize(DATA_HDR) == DATA_HDR_LEN
 
+packed_version = 'FS21'
+
 def warn(message, *data):
     LOG('ZODB FS', WARNING, "%s  warn: %s\n" % (packed_version,
                                                 (message % data)))
@@ -198,7 +193,16 @@
     pass
 
 class CorruptedDataError(CorruptedFileStorageError):
-    pass
+
+    def __init__(self, oid=None, buf=None):
+        self.oid = oid
+        self.buf = buf
+
+    def __str__(self):
+        if self.oid:
+            return "Error reading oid %016x.  Found %r" % (self.oid, self.buf)
+        else:

[-=- -=- -=- 1852 lines omitted -=- -=- -=-]

-                    # this.
+                if not h.back:
+                    # If the backpointer is 0, then this transaction
+                    # undoes the object creation.  It either aborts
+                    # the version that created the object or undid the
+                    # transaction that created it.  Return None
+                    # instead of a pickle to indicate this.
                     data = None
                 else:
-                    data, _s, tid = _loadBackTxn(self._file, oid, bp)
-                    prev_txn = getTxnFromData(self._file, oid, bp)
+                    data, _s, tid = self._loadBackTxn(h.oid, h.back)
+                    prev_txn = self.getTxnFromData(h.oid, h.back)
 
-            r = Record(oid, serial, version, data, prev_txn)
+            r = Record(h.oid, h.serial, h.version, data, prev_txn)
 
             return r
 
@@ -2480,3 +2297,30 @@
              'description': d}
         d.update(e)
         return d
+
+class DataHeader:
+    """Header for a data record."""
+
+    __slots__ = ("oid", "serial", "prev", "tloc", "vlen", "plen", "back",
+                 # These three attributes are only defined when vlen > 0
+                 "pnv", "vprev", "version")
+
+    version = ""
+    back = 0
+
+    def __init__(self, oid, serial, prev, tloc, vlen, plen):
+        self.oid = oid
+        self.serial = serial
+        self.prev = prev
+        self.tloc = tloc
+        self.vlen = vlen
+        self.plen = plen
+
+    def fromString(cls, s):
+        return cls(*struct.unpack(DATA_HDR, s))
+        
+    fromString = classmethod(fromString)
+
+    def parseVersion(self, buf):
+        self.pnv, self.vprev = struct.unpack(">QQ", buf[:16])
+        self.version = buf[16:]


=== ZODB4/ZODB/BaseStorage.py 1.25 => 1.26 ===
--- ZODB4/ZODB/BaseStorage.py:1.25	Fri Dec  6 15:18:15 2002
+++ ZODB4/ZODB/BaseStorage.py	Wed Dec 11 18:40:50 2002
@@ -16,6 +16,8 @@
 $Id$
 """
 
+__metaclass__ = type
+
 import threading
 from ZODB import POSException
 from ZODB.TimeStamp import newTimeStamp, TimeStamp