[Zodb-checkins] CVS: ZODB3/ZODB - fsdump.py:1.3.72.1.4.1 FileStorage.py:1.105.2.6.2.1
Jeremy Hylton
jeremy@zope.com
Fri, 15 Nov 2002 10:44:14 -0500
Update of /cvs-repository/ZODB3/ZODB
In directory cvs.zope.org:/tmp/cvs-serv23852/ZODB
Modified Files:
Tag: ZODB3-restore-debug-branch
fsdump.py FileStorage.py
Log Message:
Add new test checkRecoverUndoInVersion(), which fails,
and a bunch of new code for debugging that problem.
=== ZODB3/ZODB/fsdump.py 1.3.72.1 => 1.3.72.1.4.1 ===
--- ZODB3/ZODB/fsdump.py:1.3.72.1 Mon Oct 21 11:15:55 2002
+++ ZODB3/ZODB/fsdump.py Fri Nov 15 10:44:13 2002
@@ -76,3 +76,80 @@
print >> file
i += 1
iter.close()
+
+import struct
+from ZODB.FileStorage import TRANS_HDR, TRANS_HDR_LEN
+from ZODB.FileStorage import DATA_HDR, DATA_HDR_LEN
+
+def fmt(p64):
+ # Return a nicely formatted string for a packaged 64-bit value
+ return "%016x" % U64(p64)
+
+class Dumper:
+ """A very verbose dumper for debuggin FileStorage problems."""
+
+ def __init__(self, path, dest=None):
+ self.file = open(path, "rb")
+ self.dest = dest
+
+ def dump(self):
+ fid = self.file.read(4)
+ print >> self.dest, "*" * 60
+ print >> self.dest, "file identifier: %r" % fid
+ while self.dump_txn():
+ pass
+
+ def dump_txn(self):
+ pos = self.file.tell()
+ h = self.file.read(TRANS_HDR_LEN)
+ if not h:
+ return False
+ tid, stlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+ end = pos + U64(stlen)
+ print >> self.dest, "=" * 60
+ print >> self.dest, "offset: %d" % pos
+ print >> self.dest, "end pos: %d" % end
+ print >> self.dest, "transaction id: %s" % fmt(tid)
+ print >> self.dest, "trec len: %d" % U64(stlen)
+ print >> self.dest, "status: %r" % status
+ user = descr = extra = ""
+ if ul:
+ user = self.file.read(ul)
+ if dl:
+ descr = self.file.read(dl)
+ if el:
+ extra = self.file.read(el)
+ print >> self.dest, "user: %r" % user
+ print >> self.dest, "description: %r" % descr
+ print >> self.dest, "len(extra): %d" % el
+ while self.file.tell() < end:
+ self.dump_data(pos)
+ stlen2 = self.file.read(8)
+ print >> self.dest, "redundant trec len: %d" % U64(stlen2)
+ return True
+
+ def dump_data(self, tloc):
+ pos = self.file.tell()
+ h = self.file.read(DATA_HDR_LEN)
+ assert len(h) == DATA_HDR_LEN
+ oid, revid, sprev, stloc, vlen, sdlen = struct.unpack(DATA_HDR, h)
+ dlen = U64(sdlen)
+ print >> self.dest, "-" * 60
+ print >> self.dest, "offset: %d" % pos
+ print >> self.dest, "oid: %s" % fmt(oid)
+ print >> self.dest, "revid: %s" % fmt(revid)
+ print >> self.dest, "previous record offset: %d" % U64(sprev)
+ print >> self.dest, "transaction offset: %d" % U64(stloc)
+ if vlen:
+ pnv = self.file.read(8)
+ sprevdata = self.file.read(8)
+ version = self.file.read(vlen)
+ print >> self.dest, "version: %r" % version
+ print >> self.dest, "non-version data offset: %d" % U64(pnv)
+ print >> self.dest, \
+ "previous version data offset: %d" % U64(sprevdata)
+ print >> self.dest, "len(data): %d" % dlen
+ self.file.read(dlen)
+ if not dlen:
+ sbp = self.file.read(8)
+ print >> self.dest, "backpointer: %d" % U64(sbp)
=== ZODB3/ZODB/FileStorage.py 1.105.2.6 => 1.105.2.6.2.1 ===
--- ZODB3/ZODB/FileStorage.py:1.105.2.6 Thu Nov 14 12:01:45 2002
+++ ZODB3/ZODB/FileStorage.py Fri Nov 15 10:44:13 2002
@@ -826,20 +826,7 @@
# We need to write some version information if this revision is
# happening in a version.
if version:
- pnv = None
- # We need to write the position of the non-version data.
- # If the previous revision of the object was in a version,
- # then it will contain a pnv record. Otherwise, the
- # previous record is the non-version data.
- if old:
- self._file.seek(old)
- h = self._file.read(42)
- doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
- if doid != oid:
- raise CorruptedDataError, h
- # XXX assert versions match?
- if vlen > 0:
- pnv = self._file.read(8)
+ pnv = self._restore_pnv(oid, old, version, prev_pos)
if pnv:
self._tfile.write(pnv)
else:
@@ -851,19 +838,61 @@
self._tfile.write(p64(pv))
self._tvindex[version] = here
self._tfile.write(version)
- # And finally, write the data
+ # And finally, write the data or a backpointer
if data is None:
if prev_pos:
self._tfile.write(p64(prev_pos))
else:
# Write a zero backpointer, which indicates an
# un-creation transaction.
- # write a backpointer instead of data
self._tfile.write(z64)
else:
self._tfile.write(data)
finally:
self._lock_release()
+
+ def _restore_pnv(self, oid, prev, version, bp):
+ # Find a valid pnv (previous non-version) pointer for this version.
+
+ # If there is no previous record, there can't be a pnv.
+ if not prev:
+ return None
+
+ pnv = None
+
+ # Load the record pointed to be prev
+ self._file.seek(prev)
+ h = self._file.read(DATA_HDR_LEN)
+ doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
+ if doid != oid:
+ raise CorruptedDataError, h
+ # If the previous record is for a version, it must have
+ # a valid pnv.
+ if vlen > 0:
+ pnv = self._file.read(8)
+ pv = self._file.read(8)
+ v = self._file.read(vlen)
+ elif bp:
+ # XXX Not sure the following is always true:
+ # The previous record is not for this version, yet we
+ # have a backpointer to it. The current record must
+ # be an undo of an abort or commit, so the backpointer
+ # must be to a version record with a pnv.
+ self._file.seek(bp)
+ h2 = self._file.read(DATA_HDR_LEN)
+ doid2, x, y, z, vlen2, sdl = unpack(DATA_HDR, h2)
+ dl = U64(sdl)
+ if oid != doid2:
+ raise CorruptedDataError, h2
+ if vlen2 > 0:
+ pnv = self._file.read(8)
+ pv = self._file.read(8)
+ v = self._file.read(8)
+ else:
+ warn("restore could not find previous non-version data "
+ "at %d or %d" % (prev, bp))
+
+ return pnv
def supportsUndo(self):
return 1