[Zodb-checkins] CVS: ZODB3/ZODB - fsdump.py:1.3.72.2 __init__.py:1.17.2.4 FileStorage.py:1.105.2.7
Jeremy Hylton
jeremy@zope.com
Fri, 15 Nov 2002 12:19:28 -0500
Update of /cvs-repository/ZODB3/ZODB
In directory cvs.zope.org:/tmp/cvs-serv1715/ZODB
Modified Files:
Tag: ZODB3-3_1-branch
fsdump.py __init__.py FileStorage.py
Log Message:
Fix two restore() bugs.
Merge changes from short-lived ZODB3-restore-debug-branch.
Add entry to NEWS file about problem.
Bump version numbers to 3.1.1 and 2.0.1.
=== ZODB3/ZODB/fsdump.py 1.3.72.1 => 1.3.72.2 ===
--- ZODB3/ZODB/fsdump.py:1.3.72.1 Mon Oct 21 11:15:55 2002
+++ ZODB3/ZODB/fsdump.py Fri Nov 15 12:18:55 2002
@@ -76,3 +76,80 @@
print >> file
i += 1
iter.close()
+
+import struct
+from ZODB.FileStorage import TRANS_HDR, TRANS_HDR_LEN
+from ZODB.FileStorage import DATA_HDR, DATA_HDR_LEN
+
+def fmt(p64):
+ # Return a nicely formatted string for a packaged 64-bit value
+ return "%016x" % U64(p64)
+
+class Dumper:
+ """A very verbose dumper for debuggin FileStorage problems."""
+
+ def __init__(self, path, dest=None):
+ self.file = open(path, "rb")
+ self.dest = dest
+
+ def dump(self):
+ fid = self.file.read(4)
+ print >> self.dest, "*" * 60
+ print >> self.dest, "file identifier: %r" % fid
+ while self.dump_txn():
+ pass
+
+ def dump_txn(self):
+ pos = self.file.tell()
+ h = self.file.read(TRANS_HDR_LEN)
+ if not h:
+ return False
+ tid, stlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+ end = pos + U64(stlen)
+ print >> self.dest, "=" * 60
+ print >> self.dest, "offset: %d" % pos
+ print >> self.dest, "end pos: %d" % end
+ print >> self.dest, "transaction id: %s" % fmt(tid)
+ print >> self.dest, "trec len: %d" % U64(stlen)
+ print >> self.dest, "status: %r" % status
+ user = descr = extra = ""
+ if ul:
+ user = self.file.read(ul)
+ if dl:
+ descr = self.file.read(dl)
+ if el:
+ extra = self.file.read(el)
+ print >> self.dest, "user: %r" % user
+ print >> self.dest, "description: %r" % descr
+ print >> self.dest, "len(extra): %d" % el
+ while self.file.tell() < end:
+ self.dump_data(pos)
+ stlen2 = self.file.read(8)
+ print >> self.dest, "redundant trec len: %d" % U64(stlen2)
+ return True
+
+ def dump_data(self, tloc):
+ pos = self.file.tell()
+ h = self.file.read(DATA_HDR_LEN)
+ assert len(h) == DATA_HDR_LEN
+ oid, revid, sprev, stloc, vlen, sdlen = struct.unpack(DATA_HDR, h)
+ dlen = U64(sdlen)
+ print >> self.dest, "-" * 60
+ print >> self.dest, "offset: %d" % pos
+ print >> self.dest, "oid: %s" % fmt(oid)
+ print >> self.dest, "revid: %s" % fmt(revid)
+ print >> self.dest, "previous record offset: %d" % U64(sprev)
+ print >> self.dest, "transaction offset: %d" % U64(stloc)
+ if vlen:
+ pnv = self.file.read(8)
+ sprevdata = self.file.read(8)
+ version = self.file.read(vlen)
+ print >> self.dest, "version: %r" % version
+ print >> self.dest, "non-version data offset: %d" % U64(pnv)
+ print >> self.dest, \
+ "previous version data offset: %d" % U64(sprevdata)
+ print >> self.dest, "len(data): %d" % dlen
+ self.file.read(dlen)
+ if not dlen:
+ sbp = self.file.read(8)
+ print >> self.dest, "backpointer: %d" % U64(sbp)
=== ZODB3/ZODB/__init__.py 1.17.2.3 => 1.17.2.4 ===
--- ZODB3/ZODB/__init__.py:1.17.2.3 Wed Oct 23 15:46:52 2002
+++ ZODB3/ZODB/__init__.py Fri Nov 15 12:18:55 2002
@@ -12,7 +12,7 @@
#
##############################################################################
-__version__ = '3.1'
+__version__ = '3.1.1'
import sys
import cPersistence, Persistence
=== ZODB3/ZODB/FileStorage.py 1.105.2.6 => 1.105.2.7 ===
--- ZODB3/ZODB/FileStorage.py:1.105.2.6 Thu Nov 14 12:01:45 2002
+++ ZODB3/ZODB/FileStorage.py Fri Nov 15 12:18:55 2002
@@ -766,9 +766,13 @@
if vl:
self._file.read(vl + 16)
# Make sure this looks like the right data record
+ if dl == 0:
+ # This is also a backpointer. Gotta trust it.
+ return pos
if dl != len(data):
- # XXX what if this data record also has a backpointer?
- # I don't think that's possible, but I'm not sure.
+ # The expected data doesn't match what's in the
+ # backpointer. Something is wrong.
+ error("Mismatch between data and backpointer at %d", pos)
return 0
_data = self._file.read(dl)
if data != _data:
@@ -826,20 +830,7 @@
# We need to write some version information if this revision is
# happening in a version.
if version:
- pnv = None
- # We need to write the position of the non-version data.
- # If the previous revision of the object was in a version,
- # then it will contain a pnv record. Otherwise, the
- # previous record is the non-version data.
- if old:
- self._file.seek(old)
- h = self._file.read(42)
- doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
- if doid != oid:
- raise CorruptedDataError, h
- # XXX assert versions match?
- if vlen > 0:
- pnv = self._file.read(8)
+ pnv = self._restore_pnv(oid, old, version, prev_pos)
if pnv:
self._tfile.write(pnv)
else:
@@ -851,20 +842,62 @@
self._tfile.write(p64(pv))
self._tvindex[version] = here
self._tfile.write(version)
- # And finally, write the data
+ # And finally, write the data or a backpointer
if data is None:
if prev_pos:
self._tfile.write(p64(prev_pos))
else:
# Write a zero backpointer, which indicates an
# un-creation transaction.
- # write a backpointer instead of data
self._tfile.write(z64)
else:
self._tfile.write(data)
finally:
self._lock_release()
+ def _restore_pnv(self, oid, prev, version, bp):
+ # Find a valid pnv (previous non-version) pointer for this version.
+
+ # If there is no previous record, there can't be a pnv.
+ if not prev:
+ return None
+
+ pnv = None
+
+ # Load the record pointed to be prev
+ self._file.seek(prev)
+ h = self._file.read(DATA_HDR_LEN)
+ doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
+ if doid != oid:
+ raise CorruptedDataError, h
+ # If the previous record is for a version, it must have
+ # a valid pnv.
+ if vlen > 0:
+ pnv = self._file.read(8)
+ pv = self._file.read(8)
+ v = self._file.read(vlen)
+ elif bp:
+ # XXX Not sure the following is always true:
+ # The previous record is not for this version, yet we
+ # have a backpointer to it. The current record must
+ # be an undo of an abort or commit, so the backpointer
+ # must be to a version record with a pnv.
+ self._file.seek(bp)
+ h2 = self._file.read(DATA_HDR_LEN)
+ doid2, x, y, z, vlen2, sdl = unpack(DATA_HDR, h2)
+ dl = U64(sdl)
+ if oid != doid2:
+ raise CorruptedDataError, h2
+ if vlen2 > 0:
+ pnv = self._file.read(8)
+ pv = self._file.read(8)
+ v = self._file.read(8)
+ else:
+ warn("restore could not find previous non-version data "
+ "at %d or %d" % (prev, bp))
+
+ return pnv
+
def supportsUndo(self):
return 1
@@ -2182,6 +2215,17 @@
tid = h[:8]
return data, serial, tid
+def getTxnFromData(file, oid, back):
+ """Return transaction id for data at back."""
+ file.seek(U64(back))
+ h = file.read(DATA_HDR_LEN)
+ doid, serial, prev, stloc, vlen, plen = unpack(DATA_HDR, h)
+ assert oid == doid
+ tloc = U64(stloc)
+ file.seek(tloc)
+ # seek to transaction header, where tid is first 8 bytes
+ return file.read(8)
+
def _truncate(file, name, pos):
seek=file.seek
seek(0,2)
@@ -2422,7 +2466,7 @@
break
self._pos = pos + dlen
- tid = None
+ prev_txn = None
if plen:
data = self._file.read(plen)
else:
@@ -2437,8 +2481,9 @@
data = None
else:
data, _s, tid = _loadBackTxn(self._file, oid, bp)
+ prev_txn = getTxnFromData(self._file, oid, bp)
- r = Record(oid, serial, version, data, tid)
+ r = Record(oid, serial, version, data, prev_txn)
return r