[Zope3-checkins] CVS: ZODB4/ZODB - fsdump.py:1.10 TimeStamp.c:1.19 Serialize.py:1.10 POSException.py:1.19 FileStorage.py:1.112 BaseStorage.py:1.26
Jeremy Hylton
jeremy@zope.com
Wed, 11 Dec 2002 18:40:51 -0500
Update of /cvs-repository/ZODB4/ZODB
In directory cvs.zope.org:/tmp/cvs-serv19253/ZODB
Modified Files:
fsdump.py TimeStamp.c Serialize.py POSException.py
FileStorage.py BaseStorage.py
Log Message:
Merge the ZODB4-Q-branch to the trunk.
The primary effect of this change is to use take advantage of the Q
struct format code to simplify FileStorage. There were many places
where FileStorage used "8s" and then had to call u64() on the result
to convert from an 8-byte string to a Python long. This was tedious
and added lots of extra noise to the FileStorage code.
Along the way, a lot of other refactoring happened. Introduce a
FileStorageFormatter that handles most of the details of reading the
low-level FileStorage format. It returns a simply Python object that
has attributes for each of the fields in a data header. The formatter
class has been used in many places, but the changes aren't complete.
It should be used for the transaction header and for writing the data
header, but I don't know when I'll get to that.
The pack implementation was refactored a bit. Most of the logic was
moved to a separate class FileStoragePacker, because the _file
attribute is a separate file object opened on the same underlying file
as the main FileStorage.
Since old-style undo is no longer supported, all the checks for 'u'
were removed and the _sane() method is no longer necessary.
The code passes all the tests, but there are a bunch of corner cases
in pack() that should be tested more thoroughly. A bunch of branches
aren't executed during the pack tests.
A run of speed.py shows that the code is faster on the whole. It's
quicker to get the Python long directly from struct than it is to get
an 8s and call another Python function to convert it. For the speed
tests, the runs with small # of objects are slower (roughly 2x), but
the runs with large # of objects are faster (roughly 50%).
=== ZODB4/ZODB/fsdump.py 1.9 => 1.10 ===
--- ZODB4/ZODB/fsdump.py:1.9 Tue Dec 3 18:37:08 2002
+++ ZODB4/ZODB/fsdump.py Wed Dec 11 18:40:50 2002
@@ -21,6 +21,9 @@
# Return a nicely formatted string for a packaged 64-bit value
return "%016x" % u64(p64)
+def dump(path, dest=None):
+ Dumper(path, dest).dump()
+
class Dumper:
"""A very verbose dumper for debugging FileStorage problems."""
@@ -40,13 +43,13 @@
h = self.file.read(TRANS_HDR_LEN)
if not h:
return False
- tid, stlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
- end = pos + u64(stlen)
+ tid, tlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+ end = pos + tlen
print >> self.dest, "=" * 60
print >> self.dest, "offset: %d" % pos
print >> self.dest, "end pos: %d" % end
print >> self.dest, "transaction id: %s" % fmt(tid)
- print >> self.dest, "trec len: %d" % u64(stlen)
+ print >> self.dest, "trec len: %d" % tlen
print >> self.dest, "status: %r" % status
user = descr = extra = ""
if ul:
@@ -60,22 +63,21 @@
print >> self.dest, "len(extra): %d" % el
while self.file.tell() < end:
self.dump_data(pos)
- stlen2 = self.file.read(8)
- print >> self.dest, "redundant trec len: %d" % u64(stlen2)
+ tlen2 = u64(self.file.read(8))
+ print >> self.dest, "redundant trec len: %d" % tlen2
return True
def dump_data(self, tloc):
pos = self.file.tell()
h = self.file.read(DATA_HDR_LEN)
assert len(h) == DATA_HDR_LEN
- oid, revid, sprev, stloc, vlen, sdlen = struct.unpack(DATA_HDR, h)
- dlen = u64(sdlen)
+ oid, revid, prev, tloc, vlen, dlen = struct.unpack(DATA_HDR, h)
print >> self.dest, "-" * 60
print >> self.dest, "offset: %d" % pos
print >> self.dest, "oid: %s" % fmt(oid)
print >> self.dest, "revid: %s" % fmt(revid)
- print >> self.dest, "previous record offset: %d" % u64(sprev)
- print >> self.dest, "transaction offset: %d" % u64(stloc)
+ print >> self.dest, "previous record offset: %d" % prev
+ print >> self.dest, "transaction offset: %d" % tloc
if vlen:
pnv = self.file.read(8)
sprevdata = self.file.read(8)
=== ZODB4/ZODB/TimeStamp.c 1.18 => 1.19 ===
--- ZODB4/ZODB/TimeStamp.c:1.18 Mon Nov 25 16:52:27 2002
+++ ZODB4/ZODB/TimeStamp.c Wed Dec 11 18:40:50 2002
@@ -105,7 +105,10 @@
static int
TimeStamp_compare(TimeStamp *v, TimeStamp *w)
{
- return memcmp(v->data, w->data, 8);
+ int cmp = memcmp(v->data, w->data, 8);
+ if (cmp < 0) return -1;
+ if (cmp > 0) return 1;
+ return 0;
}
static long
=== ZODB4/ZODB/Serialize.py 1.9 => 1.10 ===
--- ZODB4/ZODB/Serialize.py:1.9 Tue Dec 3 18:07:05 2002
+++ ZODB4/ZODB/Serialize.py Wed Dec 11 18:40:50 2002
@@ -63,13 +63,14 @@
mod = getattr(mod, part)
return getattr(mod, name)
-def getClassMetadata(object=None, klass=None):
+def getClassMetadata(obj=None, klass=None):
if klass is None:
- klass = object.__class__
+ klass = obj.__class__
module = klass.__module__
classname = klass.__name__
- if hasattr(object, "__getnewargs__"):
- newargs = object.__getnewargs__()
+ # XXX what if obj is None and we were passed klass?
+ if hasattr(obj, "__getnewargs__"):
+ newargs = obj.__getnewargs__()
else:
newargs = None
return module, classname, newargs
@@ -177,11 +178,11 @@
def _new_object(self, module, classname, newargs=None):
klass = getClass(module, classname)
if newargs is None:
- object = klass.__new__(klass)
+ obj = klass.__new__(klass)
else:
- object = klass.__new__(klass, *newargs)
+ obj = klass.__new__(klass, *newargs)
- return object
+ return obj
def getClassName(self, pickle):
unpickler = self._get_unpickler(pickle)
@@ -206,10 +207,10 @@
def getObject(self, pickle):
unpickler = self._get_unpickler(pickle)
module, classname, newargs = unpickler.load()
- object = self._new_object(module, classname, newargs)
+ obj = self._new_object(module, classname, newargs)
state = unpickler.load()
- object.__setstate__(state)
- return object
+ obj.__setstate__(state)
+ return obj
class SimpleObjectReader(BaseObjectReader):
"""Minimal reader for a single data record."""
@@ -231,23 +232,23 @@
# Quick instance reference. We know all we need to know
# to create the instance w/o hitting the db, so go for it!
oid, classmeta = oid
- object = self._cache.get(oid)
- if object is not None:
- return object
+ obj = self._cache.get(oid)
+ if obj is not None:
+ return obj
- object = self._new_object(*classmeta)
+ obj = self._new_object(*classmeta)
# XXX should be done by connection
- object._p_oid = oid
- object._p_jar = self._conn
- object._p_changed = None
+ obj._p_oid = oid
+ obj._p_jar = self._conn
+ obj._p_changed = None
- self._cache[oid] = object
- return object
+ self._cache[oid] = obj
+ return obj
- object = self._cache.get(oid)
- if object is not None:
- return object
+ obj = self._cache.get(oid)
+ if obj is not None:
+ return obj
return self._conn[oid]
class CopyReference:
=== ZODB4/ZODB/POSException.py 1.18 => 1.19 ===
--- ZODB4/ZODB/POSException.py:1.18 Tue Dec 3 13:38:02 2002
+++ ZODB4/ZODB/POSException.py Wed Dec 11 18:40:50 2002
@@ -152,6 +152,14 @@
class VersionLockError(VersionError, TransactionError):
"""Can't modify an object that is modified in unsaved version."""
+ def __init__(self, oid, version):
+ self.oid = oid
+ self.version = version
+
+ def __str__(self):
+ return "%s locked in version %r" % (_fmt_oid(self.oid),
+ self.version)
+
class UndoError(POSError):
"""An attempt was made to undo a non-undoable transaction."""
=== ZODB4/ZODB/FileStorage.py 1.111 => 1.112 === (1852/1952 lines abridged)
--- ZODB4/ZODB/FileStorage.py:1.111 Fri Dec 6 18:09:29 2002
+++ ZODB4/ZODB/FileStorage.py Wed Dec 11 18:40:50 2002
@@ -139,28 +139,23 @@
from ZODB.TimeStamp import TimeStamp, newTimeStamp, timeStampFromTime
from ZODB.lock_file import lock_file
from ZODB.utils import p64, u64, cp, z64
-
-try:
- from ZODB.fsIndex import fsIndex
-except ImportError:
- def fsIndex():
- return {}
+from ZODB.fsIndex import fsIndex
from zLOG import LOG, BLATHER, WARNING, ERROR, PANIC
t32 = 1L << 32
# the struct formats for the headers
-TRANS_HDR = ">8s8scHHH"
-DATA_HDR = ">8s8s8s8sH8s"
+TRANS_HDR = ">8sQcHHH"
+DATA_HDR = ">8s8sQQHQ"
# constants to support various header sizes
TRANS_HDR_LEN = 23
DATA_HDR_LEN = 42
DATA_VERSION_HDR_LEN = 58
-
-packed_version='FS21'
assert struct.calcsize(TRANS_HDR) == TRANS_HDR_LEN
assert struct.calcsize(DATA_HDR) == DATA_HDR_LEN
+packed_version = 'FS21'
+
def warn(message, *data):
LOG('ZODB FS', WARNING, "%s warn: %s\n" % (packed_version,
(message % data)))
@@ -198,7 +193,16 @@
pass
class CorruptedDataError(CorruptedFileStorageError):
- pass
+
+ def __init__(self, oid=None, buf=None):
+ self.oid = oid
+ self.buf = buf
+
+ def __str__(self):
+ if self.oid:
+ return "Error reading oid %016x. Found %r" % (self.oid, self.buf)
+ else:
[-=- -=- -=- 1852 lines omitted -=- -=- -=-]
- # this.
+ if not h.back:
+ # If the backpointer is 0, then this transaction
+ # undoes the object creation. It either aborts
+ # the version that created the object or undid the
+ # transaction that created it. Return None
+ # instead of a pickle to indicate this.
data = None
else:
- data, _s, tid = _loadBackTxn(self._file, oid, bp)
- prev_txn = getTxnFromData(self._file, oid, bp)
+ data, _s, tid = self._loadBackTxn(h.oid, h.back)
+ prev_txn = self.getTxnFromData(h.oid, h.back)
- r = Record(oid, serial, version, data, prev_txn)
+ r = Record(h.oid, h.serial, h.version, data, prev_txn)
return r
@@ -2480,3 +2297,30 @@
'description': d}
d.update(e)
return d
+
+class DataHeader:
+ """Header for a data record."""
+
+ __slots__ = ("oid", "serial", "prev", "tloc", "vlen", "plen", "back",
+ # These three attributes are only defined when vlen > 0
+ "pnv", "vprev", "version")
+
+ version = ""
+ back = 0
+
+ def __init__(self, oid, serial, prev, tloc, vlen, plen):
+ self.oid = oid
+ self.serial = serial
+ self.prev = prev
+ self.tloc = tloc
+ self.vlen = vlen
+ self.plen = plen
+
+ def fromString(cls, s):
+ return cls(*struct.unpack(DATA_HDR, s))
+
+ fromString = classmethod(fromString)
+
+ def parseVersion(self, buf):
+ self.pnv, self.vprev = struct.unpack(">QQ", buf[:16])
+ self.version = buf[16:]
=== ZODB4/ZODB/BaseStorage.py 1.25 => 1.26 ===
--- ZODB4/ZODB/BaseStorage.py:1.25 Fri Dec 6 15:18:15 2002
+++ ZODB4/ZODB/BaseStorage.py Wed Dec 11 18:40:50 2002
@@ -16,6 +16,8 @@
$Id$
"""
+__metaclass__ = type
+
import threading
from ZODB import POSException
from ZODB.TimeStamp import newTimeStamp, TimeStamp