[Zope-Checkins] CVS: StandaloneZODB/ZODB - BaseStorage.py:1.16.8.2 DB.py:1.36.6.2 FileStorage.py:1.76.8.2 cPersistence.c:1.47.8.2 cPickleCache.c:1.37.8.2
Jeremy Hylton
jeremy@zope.com
Tue, 18 Dec 2001 12:14:15 -0500
Update of /cvs-repository/StandaloneZODB/ZODB
In directory cvs.zope.org:/tmp/cvs-serv31892
Modified Files:
Tag: Standby-branch
BaseStorage.py DB.py FileStorage.py cPersistence.c
cPickleCache.c
Log Message:
Merge in jeremy-Standby-branch
=== StandaloneZODB/ZODB/BaseStorage.py 1.16.8.1 => 1.16.8.2 ===
t=self._ts=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
self._serial=`t`
- if base is None: self._oid='\0\0\0\0\0\0\0\0'
- else: self._oid=base._oid
+ if base is None:
+ self._oid='\0\0\0\0\0\0\0\0'
+ else:
+ self._oid=base._oid
def abortVersion(self, src, transaction):
if transaction is not self._transaction:
@@ -232,3 +234,9 @@
self.tpc_vote(transaction)
self.tpc_finish(transaction)
+
+class TransactionRecord:
+ """Abstract base class for iterator protocol"""
+
+class DataRecord:
+ """Abstract base class for iterator protocol"""
=== StandaloneZODB/ZODB/DB.py 1.36.6.1 => 1.36.6.2 ===
=== StandaloneZODB/ZODB/FileStorage.py 1.76.8.1 => 1.76.8.2 ===
read_only=read_only,
)
+ self._ltid = tid
self._ts = tid = TimeStamp(tid)
t = time.time()
@@ -395,10 +396,15 @@
def close(self):
self._file.close()
- if hasattr(self,'_lock_file'): self._lock_file.close()
- if self._tfile: self._tfile.close()
- try: self._save_index()
- except: pass # We don't care if this fails.
+ if hasattr(self,'_lock_file'):
+ self._lock_file.close()
+ if self._tfile:
+ self._tfile.close()
+ try:
+ self._save_index()
+ except:
+ # XXX should log the error, though
+ pass # We don't care if this fails.
def commitVersion(self, src, dest, transaction, abort=None):
# We are going to commit by simply storing back pointers.
@@ -419,9 +425,8 @@
self._lock_acquire()
try:
- file=self._file
- read=file.read
- seek=file.seek
+ read=self._file.read
+ seek=self._file.seek
tfile=self._tfile
write=tfile.write
tindex=self._tindex
@@ -708,9 +713,9 @@
# We have to check lengths here because struct.pack
# doesn't raise an exception on overflow!
- if luser > 65535: raise FileStorageError, 'user name too long'
- if ldesc > 65535: raise FileStorageError, 'description too long'
- if lext > 65535: raise FileStorageError, 'too much extension data'
+ if luser > 65535: raise FileStorageError('user name too long')
+ if ldesc > 65535: raise FileStorageError('description too long')
+ if lext > 65535: raise FileStorageError('too much extension data')
tlen=self._thl
pos=self._pos
@@ -724,7 +729,7 @@
# suspect.
write(pack(
">8s" "8s" "c" "H" "H" "H"
- ,tid, stl, 'c', luser, ldesc, lext,
+ ,tid, stl,'c', luser, ldesc, lext,
))
if user: write(user)
if desc: write(desc)
@@ -760,6 +765,7 @@
self._index.update(self._tindex)
self._vindex.update(self._tvindex)
+ self._ltid = tid
def _abort(self):
if self._nextpos:
@@ -1557,8 +1563,32 @@
self._packt=z64
_lock_release()
- def iterator(self):
- return FileIterator(self._file_name)
+ def iterator(self, start=None, stop=None):
+ return FileIterator(self._file_name, start, stop)
+
+ def lastTransaction(self):
+ """Return transaction id for last committed transaction"""
+ return self._ltid
+
+ def lastSerial(self, oid):
+ """Return last serialno committed for object oid.
+
+ If there is no serialno for this oid -- which can only occur
+ if it is a new object -- return None.
+ """
+ try:
+ pos = self._index[oid]
+ except KeyError:
+ return None
+ self._file.seek(pos)
+ # first 8 bytes are oid, second 8 bytes are serialno
+ h = self._file.read(16)
+ if len(h) < 16:
+ raise CorruptedDataError, h
+ if h[:8] != oid:
+ h = h + self._file.read(26) # get rest of header
+ raise CorruptedDataError, h
+ return h[8:]
def shift_transactions_forward(index, vindex, tindex, file, pos, opos):
"""Copy transactions forward in the data file
@@ -1975,14 +2005,41 @@
"""
_ltid=z64
- def __init__(self, file):
+ def __init__(self, file, start=None, stop=None):
if isinstance(file, StringType):
- file=open(file, 'rb')
- self._file=file
- if file.read(4) != packed_version: raise FileStorageFormatError, name
+ file = open(file, 'rb')
+ self._file = file
+ if file.read(4) != packed_version:
+ raise FileStorageFormatError, name
file.seek(0,2)
- self._file_size=file.tell()
- self._pos=4L
+ self._file_size = file.tell()
+ self._pos = 4L
+ assert start is None or isinstance(start, StringType)
+ assert stop is None or isinstance(stop, StringType)
+ if start:
+ self._skip_to_start(start)
+ self._stop = stop
+
+ def _skip_to_start(self, start):
+ # Scan through the transaction records doing almost no sanity
+ # checks.
+ self._file.seek(self._pos)
+ while 1:
+ self._pos = self._file.tell()
+ h = self._file.read(16)
+ if len(h) < 16:
+ return
+ tid, stl = unpack(">8s8s", h)
+ if tid >= start:
+ return
+ tl = U64(stl)
+ self._file.seek(tl - 8, 1)
+ rtl = self._file.read(8)
+ if rtl != stl:
+ pos = self._file.tell() - 8
+ panic("%s has inconsistent transaction length at %s "
+ "(%s != %s)",
+ self._file.name, pos, rtl, stl)
def next(self, index=0):
file=self._file
@@ -1990,6 +2047,7 @@
read=file.read
pos=self._pos
+ LOG("ZODB FS", -100, "next(%d)" % index)
while 1:
# Read the transaction record
seek(pos)
@@ -2000,7 +2058,7 @@
if el < 0: el=t32-el
if tid <= self._ltid:
- warn("%s time-stamp reduction at %s", name, pos)
+ warn("%s time-stamp reduction at %s", self._file.name, pos)
self._ltid=tid
tl=U64(stl)
@@ -2010,11 +2068,12 @@
# cleared. They may also be corrupted,
# in which case, we don't want to totally lose the data.
warn("%s truncated, possibly due to damaged records at %s",
- name, pos)
+ self._file.name, pos)
break
if status not in ' up':
- warn('%s has invalid status, %s, at %s', name, status, pos)
+ warn('%s has invalid status, %s, at %s', self._file.name,
+ status, pos)
if tl < (23+ul+dl+el):
# We're in trouble. Find out if this is bad data in
@@ -2028,16 +2087,24 @@
# reasonable:
if self._file_size - rtl < pos or rtl < 23:
nearPanic('%s has invalid transaction header at %s',
- name, pos)
+ self._file.name, pos)
warn("It appears that there is invalid data at the end of "
"the file, possibly due to a system crash. %s "
"truncated to recover from bad data at end."
- % name)
+ % self._file.name)
break
else:
- warn('%s has invalid transaction header at %s', name, pos)
+ warn('%s has invalid transaction header at %s',
+ self._file.name, pos)
break
+ if self._stop is not None:
+ LOG("ZODB FS", -100,
+ ("tid %s > stop %s ? %d" %
+ (repr(tid), repr(self._stop), tid > self._stop)))
+ if self._stop is not None and tid > self._stop:
+ raise IndexError, index
+
tpos=pos
tend=tpos+tl
@@ -2047,7 +2114,7 @@
h=read(8)
if h != stl:
panic('%s has inconsistent transaction length at %s',
- name, pos)
+ self._file.name, pos)
pos=tend+8
continue
@@ -2073,7 +2140,7 @@
h=read(8)
if h != stl:
warn("%s redundant transaction length check failed at %s",
- name, pos)
+ self._file.name, pos)
break
self._pos=pos+8
@@ -2081,7 +2148,7 @@
raise IndexError, index
-class RecordIterator(Iterator):
+class RecordIterator(Iterator, BaseStorage.TransactionRecord):
"""Iterate over the transactions in a FileStorage file.
"""
def __init__(self, tid, status, user, desc, ext, pos, stuff):
@@ -2133,9 +2200,8 @@
return r
raise IndexError, index
-
-class Record:
+class Record(BaseStorage.DataRecord):
"""An abstract database record
"""
def __init__(self, *args):
=== StandaloneZODB/ZODB/cPersistence.c 1.47.8.1 => 1.47.8.2 ===
=== StandaloneZODB/ZODB/cPickleCache.c 1.37.8.1 => 1.37.8.2 ===
{
PyTuple_SET_ITEM(t, 0, v);
- v = PyObject_CallObject(self->setklassstate, t);
+ v = PyObject_Call(self->setklassstate, t, NULL);
PyTuple_SET_ITEM(t, 0, NULL);
Py_DECREF(t);
}