[Zodb-checkins] CVS: Zope/lib/python/ZODB - fsdump.py:1.3.70.2 __init__.py:1.14.2.2 FileStorage.py:1.98.2.2
Jeremy Hylton
jeremy@zope.com
Fri, 15 Nov 2002 12:45:04 -0500
Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv5028/lib/python/ZODB
Modified Files:
Tag: Zope-2_6-branch
fsdump.py __init__.py FileStorage.py
Log Message:
Fix two problems with restore() and backpointers.
Merge ZODB3-restore-debug-branch and misc changes from
ZODB3-3_1-branch.
=== Zope/lib/python/ZODB/fsdump.py 1.3.70.1 => 1.3.70.2 ===
--- Zope/lib/python/ZODB/fsdump.py:1.3.70.1 Tue Nov 12 16:13:58 2002
+++ Zope/lib/python/ZODB/fsdump.py Fri Nov 15 12:45:03 2002
@@ -76,3 +76,80 @@
print >> file
i += 1
iter.close()
+
+import struct
+from ZODB.FileStorage import TRANS_HDR, TRANS_HDR_LEN
+from ZODB.FileStorage import DATA_HDR, DATA_HDR_LEN
+
+def fmt(p64):
+ # Return a nicely formatted string for a packaged 64-bit value
+ return "%016x" % U64(p64)
+
+class Dumper:
+ """A very verbose dumper for debuggin FileStorage problems."""
+
+ def __init__(self, path, dest=None):
+ self.file = open(path, "rb")
+ self.dest = dest
+
+ def dump(self):
+ fid = self.file.read(4)
+ print >> self.dest, "*" * 60
+ print >> self.dest, "file identifier: %r" % fid
+ while self.dump_txn():
+ pass
+
+ def dump_txn(self):
+ pos = self.file.tell()
+ h = self.file.read(TRANS_HDR_LEN)
+ if not h:
+ return False
+ tid, stlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+ end = pos + U64(stlen)
+ print >> self.dest, "=" * 60
+ print >> self.dest, "offset: %d" % pos
+ print >> self.dest, "end pos: %d" % end
+ print >> self.dest, "transaction id: %s" % fmt(tid)
+ print >> self.dest, "trec len: %d" % U64(stlen)
+ print >> self.dest, "status: %r" % status
+ user = descr = extra = ""
+ if ul:
+ user = self.file.read(ul)
+ if dl:
+ descr = self.file.read(dl)
+ if el:
+ extra = self.file.read(el)
+ print >> self.dest, "user: %r" % user
+ print >> self.dest, "description: %r" % descr
+ print >> self.dest, "len(extra): %d" % el
+ while self.file.tell() < end:
+ self.dump_data(pos)
+ stlen2 = self.file.read(8)
+ print >> self.dest, "redundant trec len: %d" % U64(stlen2)
+ return True
+
+ def dump_data(self, tloc):
+ pos = self.file.tell()
+ h = self.file.read(DATA_HDR_LEN)
+ assert len(h) == DATA_HDR_LEN
+ oid, revid, sprev, stloc, vlen, sdlen = struct.unpack(DATA_HDR, h)
+ dlen = U64(sdlen)
+ print >> self.dest, "-" * 60
+ print >> self.dest, "offset: %d" % pos
+ print >> self.dest, "oid: %s" % fmt(oid)
+ print >> self.dest, "revid: %s" % fmt(revid)
+ print >> self.dest, "previous record offset: %d" % U64(sprev)
+ print >> self.dest, "transaction offset: %d" % U64(stloc)
+ if vlen:
+ pnv = self.file.read(8)
+ sprevdata = self.file.read(8)
+ version = self.file.read(vlen)
+ print >> self.dest, "version: %r" % version
+ print >> self.dest, "non-version data offset: %d" % U64(pnv)
+ print >> self.dest, \
+ "previous version data offset: %d" % U64(sprevdata)
+ print >> self.dest, "len(data): %d" % dlen
+ self.file.read(dlen)
+ if not dlen:
+ sbp = self.file.read(8)
+ print >> self.dest, "backpointer: %d" % U64(sbp)
=== Zope/lib/python/ZODB/__init__.py 1.14.2.1 => 1.14.2.2 ===
--- Zope/lib/python/ZODB/__init__.py:1.14.2.1 Tue Nov 12 16:13:58 2002
+++ Zope/lib/python/ZODB/__init__.py Fri Nov 15 12:45:03 2002
@@ -12,7 +12,7 @@
#
##############################################################################
-__version__ = '3.1'
+__version__ = '3.1.1'
import sys
import cPersistence, Persistence
=== Zope/lib/python/ZODB/FileStorage.py 1.98.2.1 => 1.98.2.2 ===
--- Zope/lib/python/ZODB/FileStorage.py:1.98.2.1 Tue Nov 5 16:23:49 2002
+++ Zope/lib/python/ZODB/FileStorage.py Fri Nov 15 12:45:03 2002
@@ -309,9 +309,6 @@
# hook to use something other than builtin dict
return {}, {}, {}, {}
- def abortVersion(self, src, transaction):
- return self.commitVersion(src, '', transaction, abort=1)
-
def _save_index(self):
"""Write the database index to a file to support quick startup
"""
@@ -441,6 +438,9 @@
# XXX should log the error, though
pass # We don't care if this fails.
+ def abortVersion(self, src, transaction):
+ return self.commitVersion(src, '', transaction, abort=1)
+
def commitVersion(self, src, dest, transaction, abort=None):
# We are going to commit by simply storing back pointers.
if self._is_read_only:
@@ -521,6 +521,9 @@
here += heredelta
current_oids[oid] = 1
+ # Once we've found the data we are looking for,
+ # we can stop chasing backpointers.
+ break
else:
# Hm. This is a non-current record. Is there a
@@ -763,9 +766,13 @@
if vl:
self._file.read(vl + 16)
# Make sure this looks like the right data record
+ if dl == 0:
+ # This is also a backpointer. Gotta trust it.
+ return pos
if dl != len(data):
- # XXX what if this data record also has a backpointer?
- # I don't think that's possible, but I'm not sure.
+ # The expected data doesn't match what's in the
+ # backpointer. Something is wrong.
+ error("Mismatch between data and backpointer at %d", pos)
return 0
_data = self._file.read(dl)
if data != _data:
@@ -823,20 +830,7 @@
# We need to write some version information if this revision is
# happening in a version.
if version:
- pnv = None
- # We need to write the position of the non-version data.
- # If the previous revision of the object was in a version,
- # then it will contain a pnv record. Otherwise, the
- # previous record is the non-version data.
- if old:
- self._file.seek(old)
- h = self._file.read(42)
- doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
- if doid != oid:
- raise CorruptedDataError, h
- # XXX assert versions match?
- if vlen > 0:
- pnv = self._file.read(8)
+ pnv = self._restore_pnv(oid, old, version, prev_pos)
if pnv:
self._tfile.write(pnv)
else:
@@ -848,20 +842,62 @@
self._tfile.write(p64(pv))
self._tvindex[version] = here
self._tfile.write(version)
- # And finally, write the data
+ # And finally, write the data or a backpointer
if data is None:
if prev_pos:
self._tfile.write(p64(prev_pos))
else:
# Write a zero backpointer, which indicates an
# un-creation transaction.
- # write a backpointer instead of data
self._tfile.write(z64)
else:
self._tfile.write(data)
finally:
self._lock_release()
+ def _restore_pnv(self, oid, prev, version, bp):
+ # Find a valid pnv (previous non-version) pointer for this version.
+
+ # If there is no previous record, there can't be a pnv.
+ if not prev:
+ return None
+
+ pnv = None
+
+ # Load the record pointed to be prev
+ self._file.seek(prev)
+ h = self._file.read(DATA_HDR_LEN)
+ doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
+ if doid != oid:
+ raise CorruptedDataError, h
+ # If the previous record is for a version, it must have
+ # a valid pnv.
+ if vlen > 0:
+ pnv = self._file.read(8)
+ pv = self._file.read(8)
+ v = self._file.read(vlen)
+ elif bp:
+ # XXX Not sure the following is always true:
+ # The previous record is not for this version, yet we
+ # have a backpointer to it. The current record must
+ # be an undo of an abort or commit, so the backpointer
+ # must be to a version record with a pnv.
+ self._file.seek(bp)
+ h2 = self._file.read(DATA_HDR_LEN)
+ doid2, x, y, z, vlen2, sdl = unpack(DATA_HDR, h2)
+ dl = U64(sdl)
+ if oid != doid2:
+ raise CorruptedDataError, h2
+ if vlen2 > 0:
+ pnv = self._file.read(8)
+ pv = self._file.read(8)
+ v = self._file.read(8)
+ else:
+ warn("restore could not find previous non-version data "
+ "at %d or %d" % (prev, bp))
+
+ return pnv
+
def supportsUndo(self):
return 1
@@ -2155,7 +2191,8 @@
doid, serial, prev, tloc, vlen, plen = unpack(DATA_HDR, h)
if vlen:
- file.seek(vlen + 16, 1)
+ file.read(16)
+ version = file.read(vlen)
if plen != z64:
return file.read(U64(plen)), serial, old, tloc
back = file.read(8) # We got a back pointer!
@@ -2178,6 +2215,17 @@
tid = h[:8]
return data, serial, tid
+def getTxnFromData(file, oid, back):
+ """Return transaction id for data at back."""
+ file.seek(U64(back))
+ h = file.read(DATA_HDR_LEN)
+ doid, serial, prev, stloc, vlen, plen = unpack(DATA_HDR, h)
+ assert oid == doid
+ tloc = U64(stloc)
+ file.seek(tloc)
+ # seek to transaction header, where tid is first 8 bytes
+ return file.read(8)
+
def _truncate(file, name, pos):
seek=file.seek
seek(0,2)
@@ -2407,29 +2455,35 @@
else:
version = ''
+ datapos = pos + DATA_HDR_LEN
+ if vlen:
+ datapos += 16 + vlen
+ assert self._file.tell() == datapos, (self._file.tell(), datapos)
+
if pos + dlen > self._tend or tloc != self._tpos:
warn("%s data record exceeds transaction record at %s",
file.name, pos)
break
self._pos = pos + dlen
- tid = None
+ prev_txn = None
if plen:
- p = self._file.read(plen)
+ data = self._file.read(plen)
else:
- p = self._file.read(8)
- if p == z64:
+ bp = self._file.read(8)
+ if bp == z64:
# If the backpointer is 0 (encoded as z64), then
# this transaction undoes the object creation. It
# either aborts the version that created the
# object or undid the transaction that created it.
# Return None instead of a pickle to indicate
# this.
- p = None
+ data = None
else:
- p, _s, tid = _loadBackTxn(self._file, oid, p)
+ data, _s, tid = _loadBackTxn(self._file, oid, bp)
+ prev_txn = getTxnFromData(self._file, oid, bp)
- r = Record(oid, serial, version, p, tid)
+ r = Record(oid, serial, version, data, prev_txn)
return r