[Zodb-checkins] CVS: Zope3/src/zodb/storage - file.py:1.8.4.6
Jeremy Hylton
jeremy@zope.com
Mon, 10 Mar 2003 18:33:44 -0500
Update of /cvs-repository/Zope3/src/zodb/storage
In directory cvs.zope.org:/tmp/cvs-serv32018/storage
Modified Files:
Tag: opaque-pickles-branch
file.py
Log Message:
Many (mostly) small changes to fix refs handling.
Make sure the references are always read before the data.
Add use of _read_data_header() in a few more places.
_loadBack_impl() returns the refs, too, since any time you read data
from a backpointer, you'd also need the refs from the backpointer.
The only case that's come up so far is an iterator's .refs attr.
Fix off-by-8 bug in _txn_undo_write(). The end of the data records
is 8 bytes before the end of the txn record.
Rename _transactionalUndoRecord() to _undo_record() and have it
return refs along with data. XXX The conflict resolution case
is not right yet.
Fix DataHeader.recordlen() to account for the version length.
=== Zope3/src/zodb/storage/file.py 1.8.4.5 => 1.8.4.6 ===
--- Zope3/src/zodb/storage/file.py:1.8.4.5 Mon Mar 10 15:59:10 2003
+++ Zope3/src/zodb/storage/file.py Mon Mar 10 18:33:43 2003
@@ -353,7 +353,6 @@
if h.version:
vindex[h.version] = pos
- dlen += 16 + len(h.version)
if pos + dlen > tend or h.tloc != tpos:
if recover:
@@ -458,27 +457,28 @@
# If backpointer is 0, object does not currently exist.
raise POSKeyError(oid)
h = self._read_data_header(back)
- self._file.read(h.nrefs * 8)
+ refs = self._file.read(h.nrefs * 8)
if h.plen:
- return self._file.read(h.plen), h.serial, back, h.tloc
+ return self._file.read(h.plen), refs, h.serial, back, h.tloc
back = h.back
def _loadBack(self, oid, back):
- data, serial, old, tloc = self._loadBack_impl(oid, back)
+ data, refs, serial, old, tloc = self._loadBack_impl(oid, back)
return data, serial
def _loadBackPOS(self, oid, back):
"""Return position of data record for backpointer."""
- data, serial, old, tloc = self._loadBack_impl(oid, back)
+ data, refs, serial, old, tloc = self._loadBack_impl(oid, back)
return old
def _loadBackTxn(self, oid, back):
"""Return data, serial, and txn id for backpointer."""
- data, serial, old, tloc = self._loadBack_impl(oid, back)
+ data, refs, serial, old, tloc = self._loadBack_impl(oid, back)
self._file.seek(tloc)
h = self._file.read(TRANS_HDR_LEN)
tid = h[:8]
- return data, serial, tid
+ refs = splitrefs(refs)
+ return data, refs, serial, tid
def getTxnFromData(self, oid, back):
"""Return transaction id for data at back."""
@@ -893,33 +893,29 @@
h = self._file.read(TRANS_HDR_LEN)
tid, tl, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
self._file.read(ul + dl + el)
- tend = tpos + tl + 8
+ tend = tpos + tl
pos = self._file.tell()
while pos < tend:
- h = self._file.read(DATA_HDR_LEN)
- _oid, serial, prev, tpos, vl, nr, dl = struct.unpack(DATA_HDR, h)
- reclen = DATA_HDR_LEN + vl + dl + (nr * 8)
- if vl:
- reclen += 16
- if _oid == oid:
- if vl:
- self._file.read(vl + 16)
+ h = self._read_data_header(pos)
+ if h.oid == oid:
+ if h.version:
+ self._file.read(h.vlen + 16)
# Read past any references data
- self._file.read(nr * 8)
+ self._file.read(h.nrefs * 8)
# Make sure this looks like the right data record
- if dl == 0:
+ if h.plen == 0:
# This is also a backpointer. Gotta trust it.
return pos
- if dl != len(data):
+ if h.plen != len(data):
# The expected data doesn't match what's in the
# backpointer. Something is wrong.
error("Mismatch between data and backpointer at %d", pos)
return 0
- _data = self._file.read(dl)
+ _data = self._file.read(h.plen)
if data != _data:
return 0
return pos
- pos += reclen
+ pos += h.recordlen()
self._file.seek(pos)
return 0
@@ -1181,11 +1177,11 @@
assert oid == h[:8]
return h[8:]
- def _transactionalUndoRecord(self, oid, pos, serial, pre, version):
- """Get the indo information for a data record
+ def _undo_record(self, h, pos):
+ """Get the undo information for a data record
- Return a 5-tuple consisting of a pickle, data pointer,
- version, packed non-version data pointer, and current
+ Return a 6-tuple consisting of a pickle, references, data
+ pointer, version, packed non-version data pointer, and current
position. If the pickle is true, then the data pointer must
be 0, but the pickle can be empty *and* the pointer 0.
"""
@@ -1193,17 +1189,17 @@
copy = 1 # Can we just copy a data pointer
# First check if it is possible to undo this record.
- tpos = self._tindex.get(oid, 0)
- ipos = self._index.get(oid, 0)
+ tpos = self._tindex.get(h.oid, 0)
+ ipos = self._index.get(h.oid, 0)
tipos = tpos or ipos
if tipos != pos:
# Eek, a later transaction modified the data, but,
# maybe it is pointing at the same data we are.
cserial, cdataptr, cdata, cver = self._undoDataInfo(
- oid, ipos, tpos)
+ h.oid, ipos, tpos)
# Versions of undone record and current record *must* match!
- if cver != version:
+ if cver != h.version:
raise UndoError(oid, 'Current and undone versions differ')
if cdataptr != pos:
@@ -1214,46 +1210,51 @@
cdataptr == tipos
or
# Backpointers are different
- self._loadBackPOS(oid, pos) !=
- self._loadBackPOS(oid, cdataptr)
+ self._loadBackPOS(h.oid, pos) !=
+ self._loadBackPOS(h.oid, cdataptr)
):
- if pre and not tpos:
+ if h.prev and not tpos:
copy = 0 # we'll try to do conflict resolution
else:
# We bail if:
# - We don't have a previous record, which should
# be impossible.
- raise UndoError(oid, "No previous record")
+ raise UndoError(h.oid, "No previous record")
except KeyError:
# LoadBack gave us a key error. Bail.
- raise UndoError(oid, "_loadBack() failed")
+ raise UndoError(h.oid, "_loadBack() failed")
# Return the data that should be written in the undo record.
- if not pre:
+ if not h.prev:
# There is no previous revision, because the object creation
# is being undone.
- return '', 0, '', '', ipos
+ return "", "", 0, "", "", ipos
- version, snv = self._getVersion(oid, pre)
+ version, snv = self._getVersion(h.oid, h.prev)
if copy:
# we can just copy our previous-record pointer forward
- return '', pre, version, snv, ipos
+ return "", "", h.prev, version, snv, ipos
try:
# returns data, serial tuple
- bdata = self._loadBack(oid, pre)[0]
+ bdata = self._loadBack(h.oid, h.prev)[0]
except KeyError:
# couldn't find oid; what's the real explanation for this?
- raise UndoError(oid, "_loadBack() failed")
+ raise UndoError(h.oid, "_loadBack() failed")
+
+
+ # XXX conflict resolution needs to give us new references, but
+ # that code isn't written yet
+
try:
- data = self.resolveConflict(oid, cserial, serial, bdata, cdata)
+ data = self.resolveConflict(h.oid, cserial, h.serial, bdata, cdata)
except interfaces.ConflictError:
data = None
if data is None:
- raise UndoError(oid,
+ raise UndoError(h.oid,
"Some data were modified by a later transaction")
- return data, 0, version, snv, ipos
+ return data, "", 0, h.version, snv, ipos
# undoLog() returns a description dict that includes an id entry.
@@ -1354,32 +1355,23 @@
failed = failures.has_key
# Read the data records for this transaction
while pos < tend:
- self._file.seek(pos)
- h = self._file.read(DATA_HDR_LEN)
- oid, serial, prev, tloc, vlen, nrefs, plen = struct.unpack(
- DATA_HDR, h)
- if failed(oid):
- del failures[oid] # second chance!
- if vlen:
- dlen = DATA_VERSION_HDR_LEN + vlen + (plen or 8) + (nrefs * 8)
- self._file.seek(16, 1)
- version = self._file.read(vlen)
- else:
- dlen = DATA_HDR_LEN + (plen or 8)
- version = ''
-
- refsdata = self._file.read(nrefs * 8)
+ h = self._read_data_header(pos)
+ if failed(h.oid):
+ del failures[h.oid] # second chance!
+ if h.version:
+ self._file.read(16 + h.vlen)
+ self._file.read(h.nrefs * 8)
try:
- p, prev, v, snv, ipos = self._transactionalUndoRecord(
- oid, pos, serial, prev, version)
+ p, refs, prev, v, snv, ipos = self._undo_record(h, pos)
except UndoError, v:
# Don't fail right away. We may be redeemed later!
- failures[oid] = v
+ failures[h.oid] = v
else:
plen = len(p)
+ nrefs = len(refs) / 8
self._tfile.write(pack(DATA_HDR,
- oid, self._serial, ipos,
+ h.oid, self._serial, ipos,
otloc, len(v), nrefs, plen))
if v:
vprev = self._tvindex.get(v, 0) or self._vindex.get(v, 0)
@@ -1389,15 +1381,14 @@
else:
odlen = DATA_HDR_LEN + (plen or 8)
- self._tfile.write(refsdata)
-
+ self._tfile.write(refs)
if p:
self._tfile.write(p)
else:
self._tfile.write(p64(prev))
- tindex[oid] = here
+ tindex[h.oid] = here
here += odlen + nrefs * 8
- pos += dlen
+ pos += h.recordlen()
if pos > tend:
raise UndoError(None, "non-undoable transaction")
if failures:
@@ -1405,7 +1396,6 @@
return tindex
-
def versionEmpty(self, version):
if not version:
# The interface is silent on this case. I think that this should
@@ -1702,7 +1692,6 @@
h = self._read_data_header(pos)
dlen = h.recordlen()
if h.version:
- dlen += 16 + h.vlen
if packing and pindex.get(oid, 0) != pos:
# This is not the most current record, or the
# oid is no longer referenced so skip it.
@@ -2066,8 +2055,6 @@
# Read the data records for this transaction
h = self._read_data_header(pos)
dlen = h.recordlen()
- if h.version:
- dlen += 16 + h.vlen
if pos + dlen > self._tend or h.tloc != self._tpos:
warn("%s data record exceeds transaction record at %s",
file.name, pos)
@@ -2076,10 +2063,9 @@
pos += dlen
prev_txn = None
- refsdata = self._file.read(h.nrefs * 8)
- refs = splitrefs(refsdata)
-
if h.plen:
+ refsdata = self._file.read(h.nrefs * 8)
+ refs = splitrefs(refsdata)
data = self._file.read(h.plen)
else:
if not h.back:
@@ -2089,8 +2075,9 @@
# transaction that created it. Return None
# instead of a pickle to indicate this.
data = None
+ refs = []
else:
- data, _s, tid = self._loadBackTxn(h.oid, h.back)
+ data, refs, _s, tid = self._loadBackTxn(h.oid, h.back)
prev_txn = self.getTxnFromData(h.oid, h.back)
yield Record(h.oid, h.serial, h.version, data, prev_txn, refs)
@@ -2180,6 +2167,7 @@
self.serial = serial
self.prev = prev
self.tloc = tloc
+
self.vlen = vlen
self.nrefs = nrefs
self.plen = plen
@@ -2194,7 +2182,10 @@
self.version = buf[16:]
def recordlen(self):
- return DATA_HDR_LEN + (self.nrefs * 8) + (self.plen or 8)
+ rlen = DATA_HDR_LEN + (self.nrefs * 8) + (self.plen or 8)
+ if self.version:
+ rlen += 16 + self.vlen
+ return rlen
def cleanup(filename):