[Zodb-checkins] CVS: StandaloneZODB/ZODB - BaseStorage.py:1.17 Connection.py:1.62 DemoStorage.py:1.9 FileStorage.py:1.77 MappingStorage.py:1.5 POSException.py:1.9
Jeremy Hylton
jeremy@zope.com
Thu, 17 Jan 2002 12:34:33 -0500
Update of /cvs-repository/StandaloneZODB/ZODB
In directory cvs.zope.org:/tmp/cvs-serv20058
Modified Files:
BaseStorage.py Connection.py DemoStorage.py FileStorage.py
MappingStorage.py POSException.py
Log Message:
Merge Standby-branch to trunk (mostly).
The Standby-branch was branched from the StandaloneZODB-1_0-branch,
which includes the BTrees-fsIndex code. I didn't include that change
in the merge, but everything else. Terse summary follows:
BaseStorage.py:
Add read-only storage feature.
Add TransactionRecord and DataRecord marker classes for iteration.
Reformat some lines.
FileStorage.py:
Add read-only storage feature.
Greg Ward's ConflictError patch
Reformat some lines.
Add lastTransaction(), lastSerialno().
Add bounds support to iterator().
Use TransactionRecord and DataRecord.
Connection.py:
DemoStorage.py:
MappingStorage.py:
Greg Ward's ConflictError patch
POSException.py:
Greg Ward's ConflictError patch
Add ReadOnlyError.
=== StandaloneZODB/ZODB/BaseStorage.py 1.16 => 1.17 ===
_serial=z64 # Transaction serial number
_tstatus=' ' # Transaction status, used for copying data
+ _is_read_only = 0
def __init__(self, name, base=None):
@@ -42,8 +43,10 @@
t=time.time()
t=self._ts=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
self._serial=`t`
- if base is None: self._oid='\0\0\0\0\0\0\0\0'
- else: self._oid=base._oid
+ if base is None:
+ self._oid='\0\0\0\0\0\0\0\0'
+ else:
+ self._oid=base._oid
def abortVersion(self, src, transaction):
if transaction is not self._transaction:
@@ -55,15 +58,24 @@
raise POSException.StorageTransactionError(self, transaction)
return []
- def close(self): pass
+ def close(self):
+ pass
- def getName(self): return self.__name__
- def getSize(self): return len(self)*300 # WAG!
- def history(self, oid, version, length=1): pass
+ def getName(self):
+ return self.__name__
+
+ def getSize(self):
+ return len(self)*300 # WAG!
+
+ def history(self, oid, version, length=1):
+ pass
- def modifiedInVersion(self, oid): return ''
+ def modifiedInVersion(self, oid):
+ return ''
def new_oid(self, last=None):
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
if last is None:
self._lock_acquire()
try:
@@ -79,10 +91,17 @@
if d < 255: return last[:-1]+chr(d+1)+'\0'*(8-len(last))
else: return self.new_oid(last[:-1])
- def registerDB(self, db, limit): pass # we don't care
+ def registerDB(self, db, limit):
+ pass # we don't care
- def supportsUndo(self): return 0
- def supportsVersions(self): return 0
+ def isReadOnly(self):
+ return self._is_read_only
+
+ def supportsUndo(self):
+ return 0
+
+ def supportsVersions(self):
+ return 0
def tpc_abort(self, transaction):
self._lock_acquire()
@@ -171,15 +190,22 @@
pass
def undo(self, transaction_id):
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
raise POSException.UndoError, 'non-undoable transaction'
- def undoLog(self, first, last, filter=None): return ()
+ def undoLog(self, first, last, filter=None):
+ return ()
- def versionEmpty(self, version): return 1
+ def versionEmpty(self, version):
+ return 1
- def versions(self, max=None): return ()
+ def versions(self, max=None):
+ return ()
- def pack(self, t, referencesf): pass
+ def pack(self, t, referencesf):
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
def getSerial(self, oid):
self._lock_acquire()
@@ -232,3 +258,9 @@
self.tpc_vote(transaction)
self.tpc_finish(transaction)
+
+class TransactionRecord:
+ """Abstract base class for iterator protocol"""
+
+class DataRecord:
+ """Abstract base class for iterator protocol"""
=== StandaloneZODB/ZODB/Connection.py 1.61 => 1.62 ===
from cPickleCache import PickleCache
-from POSException import ConflictError
+from POSException import ConflictError, ReadConflictError
from ExtensionClass import Base
import ExportImport, TmpStore
from zLOG import LOG, ERROR, BLATHER
@@ -248,7 +248,7 @@
or
invalid(None)
):
- raise ConflictError, `oid`
+ raise ConflictError(object=object)
self._invalidating.append(oid)
else:
@@ -315,7 +315,7 @@
or
invalid(None)
):
- raise ConflictError, `oid`
+ raise ConflictError(object=object)
self._invalidating.append(oid)
klass = object.__class__
@@ -459,7 +459,7 @@
if invalid(oid) or invalid(None):
if not hasattr(object.__class__, '_p_independent'):
get_transaction().register(self)
- raise ConflictError(`oid`, `object.__class__`)
+ raise ReadConflictError(object=object)
invalid=1
else:
invalid=0
@@ -484,7 +484,7 @@
except KeyError: pass
else:
get_transaction().register(self)
- raise ConflictError(`oid`, `object.__class__`)
+ raise ConflictError(object=object)
except ConflictError:
raise
@@ -544,7 +544,7 @@
def tpc_begin(self, transaction, sub=None):
if self._invalid(None): # Some nitwit invalidated everything!
- raise ConflictError, "transaction already invalidated"
+ raise ConflictError("transaction already invalidated")
self._invalidating=[]
self._creating=[]
=== StandaloneZODB/ZODB/DemoStorage.py 1.8 => 1.9 ===
nv=old
- if serial != oserial: raise POSException.ConflictError
+ if serial != oserial:
+ raise POSException.ConflictError(serials=(oserial, serial))
serial=self._serial
r=[oid, serial, old, version and (version, nv) or None, data]
=== StandaloneZODB/ZODB/FileStorage.py 1.76 => 1.77 ===
__version__='$Revision$'[11:-2]
-import struct, time, os, bpthread, string, base64, sys
+import struct, time, os, string, base64, sys
from struct import pack, unpack
-from cPickle import loads
import POSException
from POSException import UndoError
from TimeStamp import TimeStamp
from lock_file import lock_file
from utils import t32, p64, U64, cp
-from zLOG import LOG, WARNING, ERROR, PANIC, register_subsystem
+from zLOG import LOG, BLATHER, WARNING, ERROR, PANIC, register_subsystem
register_subsystem('ZODB FS')
import BaseStorage
-from cPickle import Pickler, Unpickler
+from cPickle import Pickler, Unpickler, loads
import ConflictResolution
try: from posix import fsync
@@ -185,6 +184,7 @@
create = 1
if read_only:
+ self._is_read_only = 1
if create:
raise ValueError, "can\'t create a read-only file"
elif stop is not None:
@@ -242,6 +242,7 @@
self._file, file_name, index, vindex, tindex, stop,
read_only=read_only,
)
+ self._ltid = tid
self._ts = tid = TimeStamp(tid)
t = time.time()
@@ -262,7 +263,8 @@
self._index_get=index.get
self._vindex_get=vindex.get
- def __len__(self): return len(self._index)
+ def __len__(self):
+ return len(self._index)
def _newIndexes(self):
# hook to use something other than builtin dict
@@ -389,13 +391,20 @@
def close(self):
self._file.close()
- if hasattr(self,'_lock_file'): self._lock_file.close()
- if self._tfile: self._tfile.close()
- try: self._save_index()
- except: pass # We don't care if this fails.
+ if hasattr(self,'_lock_file'):
+ self._lock_file.close()
+ if self._tfile:
+ self._tfile.close()
+ try:
+ self._save_index()
+ except:
+ # XXX should log the error, though
+ pass # We don't care if this fails.
def commitVersion(self, src, dest, transaction, abort=None):
# We are going to commit by simply storing back pointers.
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
if not (src and isinstance(src, StringType)
and isinstance(dest, StringType)):
raise POSException.VersionCommitError('Invalid source version')
@@ -413,9 +422,8 @@
self._lock_acquire()
try:
- file=self._file
- read=file.read
- seek=file.seek
+ read=self._file.read
+ seek=self._file.seek
tfile=self._tfile
write=tfile.write
tindex=self._tindex
@@ -607,6 +615,8 @@
finally: self._lock_release()
def store(self, oid, serial, data, version, transaction):
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
if transaction is not self._transaction:
raise POSException.StorageTransactionError(self, transaction)
@@ -632,8 +642,8 @@
if serial != oserial:
data=self.tryToResolveConflict(oid, oserial, serial, data)
if not data:
- raise POSException.ConflictError, (
- serial, oserial)
+ raise POSException.ConflictError(oid=oid,
+ serials=(oserial, serial))
else:
oserial=serial
@@ -672,13 +682,17 @@
finally:
self._lock_release()
- def supportsUndo(self): return 1
- def supportsVersions(self): return 1
+ def supportsUndo(self):
+ return 1
+
+ def supportsVersions(self):
+ return 1
def _clear_temp(self):
self._tindex.clear()
self._tvindex.clear()
- self._tfile.seek(0)
+ if self._tfile is not None:
+ self._tfile.seek(0)
def _begin(self, tid, u, d, e):
self._thl=23+len(u)+len(d)+len(e)
@@ -702,9 +716,9 @@
# We have to check lengths here because struct.pack
# doesn't raise an exception on overflow!
- if luser > 65535: raise FileStorageError, 'user name too long'
- if ldesc > 65535: raise FileStorageError, 'description too long'
- if lext > 65535: raise FileStorageError, 'too much extension data'
+ if luser > 65535: raise FileStorageError('user name too long')
+ if ldesc > 65535: raise FileStorageError('description too long')
+ if lext > 65535: raise FileStorageError('too much extension data')
tlen=self._thl
pos=self._pos
@@ -718,7 +732,7 @@
# suspect.
write(pack(
">8s" "8s" "c" "H" "H" "H"
- ,tid, stl, 'c', luser, ldesc, lext,
+ ,tid, stl,'c', luser, ldesc, lext,
))
if user: write(user)
if desc: write(desc)
@@ -754,6 +768,7 @@
self._index.update(self._tindex)
self._vindex.update(self._tvindex)
+ self._ltid = tid
def _abort(self):
if self._nextpos:
@@ -761,6 +776,8 @@
self._nextpos=0
def undo(self, transaction_id):
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
self._lock_acquire()
try:
self._clear_index()
@@ -808,7 +825,8 @@
return t.keys()
finally: self._lock_release()
- def supportsTransactionalUndo(self): return 1
+ def supportsTransactionalUndo(self):
+ return 1
def _undoDataInfo(self, oid, pos, tpos):
"""Return the serial, data pointer, data, and version for the oid
@@ -930,7 +948,9 @@
# writing a file position rather than a pickle. Sometimes, we
# may do conflict resolution, in which case we actually copy
# new data that results from resolution.
-
+
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
if transaction is not self._transaction:
raise POSException.StorageTransactionError(self, transaction)
@@ -1175,6 +1195,8 @@
the associated data are copied, since the old records are not copied.
"""
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
# Ugh, this seems long
packing=1 # are we in the packing phase (or the copy phase)
@@ -1551,8 +1573,32 @@
self._packt=z64
_lock_release()
- def iterator(self):
- return FileIterator(self._file_name)
+ def iterator(self, start=None, stop=None):
+ return FileIterator(self._file_name, start, stop)
+
+ def lastTransaction(self):
+ """Return transaction id for last committed transaction"""
+ return self._ltid
+
+ def lastSerial(self, oid):
+ """Return last serialno committed for object oid.
+
+ If there is no serialno for this oid -- which can only occur
+ if it is a new object -- return None.
+ """
+ try:
+ pos = self._index[oid]
+ except KeyError:
+ return None
+ self._file.seek(pos)
+ # first 8 bytes are oid, second 8 bytes are serialno
+ h = self._file.read(16)
+ if len(h) < 16:
+ raise CorruptedDataError, h
+ if h[:8] != oid:
+ h = h + self._file.read(26) # get rest of header
+ raise CorruptedDataError, h
+ return h[8:]
def shift_transactions_forward(index, vindex, tindex, file, pos, opos):
"""Copy transactions forward in the data file
@@ -1969,14 +2015,46 @@
"""
_ltid=z64
- def __init__(self, file):
+ def __init__(self, file, start=None, stop=None):
if isinstance(file, StringType):
- file=open(file, 'rb')
- self._file=file
- if file.read(4) != packed_version: raise FileStorageFormatError, name
+ file = open(file, 'rb')
+ self._file = file
+ if file.read(4) != packed_version:
+ raise FileStorageFormatError, name
file.seek(0,2)
- self._file_size=file.tell()
- self._pos=4L
+ self._file_size = file.tell()
+ self._pos = 4L
+ assert start is None or isinstance(start, StringType)
+ assert stop is None or isinstance(stop, StringType)
+ if start:
+ self._skip_to_start(start)
+ self._stop = stop
+
+ def _skip_to_start(self, start):
+ # Scan through the transaction records doing almost no sanity
+ # checks.
+ while 1:
+ self._file.seek(self._pos)
+ h = self._file.read(16)
+ if len(h) < 16:
+ return
+ tid, stl = unpack(">8s8s", h)
+ if tid >= start:
+ return
+ tl = U64(stl)
+ try:
+ self._pos += tl + 8
+ except OverflowError:
+ self._pos = long(self._pos) + tl + 8
+ if __debug__:
+ # Sanity check
+ self._file.seek(self._pos - 8, 0)
+ rtl = self._file.read(8)
+ if rtl != stl:
+ pos = self._file.tell() - 8
+ panic("%s has inconsistent transaction length at %s "
+ "(%s != %s)",
+ self._file.name, pos, U64(rtl), U64(stl))
def next(self, index=0):
file=self._file
@@ -1984,6 +2062,7 @@
read=file.read
pos=self._pos
+ LOG("ZODB FS", BLATHER, "next(%d)" % index)
while 1:
# Read the transaction record
seek(pos)
@@ -1994,7 +2073,7 @@
if el < 0: el=t32-el
if tid <= self._ltid:
- warn("%s time-stamp reduction at %s", name, pos)
+ warn("%s time-stamp reduction at %s", self._file.name, pos)
self._ltid=tid
tl=U64(stl)
@@ -2004,11 +2083,12 @@
# cleared. They may also be corrupted,
# in which case, we don't want to totally lose the data.
warn("%s truncated, possibly due to damaged records at %s",
- name, pos)
+ self._file.name, pos)
break
if status not in ' up':
- warn('%s has invalid status, %s, at %s', name, status, pos)
+ warn('%s has invalid status, %s, at %s', self._file.name,
+ status, pos)
if tl < (23+ul+dl+el):
# We're in trouble. Find out if this is bad data in
@@ -2022,16 +2102,24 @@
# reasonable:
if self._file_size - rtl < pos or rtl < 23:
nearPanic('%s has invalid transaction header at %s',
- name, pos)
+ self._file.name, pos)
warn("It appears that there is invalid data at the end of "
"the file, possibly due to a system crash. %s "
"truncated to recover from bad data at end."
- % name)
+ % self._file.name)
break
else:
- warn('%s has invalid transaction header at %s', name, pos)
+ warn('%s has invalid transaction header at %s',
+ self._file.name, pos)
break
+ if self._stop is not None:
+ LOG("ZODB FS", BLATHER,
+ ("tid %x > stop %x ? %d" %
+ (U64(tid), U64(self._stop), tid > self._stop)))
+ if self._stop is not None and tid > self._stop:
+ raise IndexError, index
+
tpos=pos
tend=tpos+tl
@@ -2041,7 +2129,7 @@
h=read(8)
if h != stl:
panic('%s has inconsistent transaction length at %s',
- name, pos)
+ self._file.name, pos)
pos=tend+8
continue
@@ -2067,7 +2155,7 @@
h=read(8)
if h != stl:
warn("%s redundant transaction length check failed at %s",
- name, pos)
+ self._file.name, pos)
break
self._pos=pos+8
@@ -2075,7 +2163,7 @@
raise IndexError, index
-class RecordIterator(Iterator):
+class RecordIterator(Iterator, BaseStorage.TransactionRecord):
"""Iterate over the transactions in a FileStorage file.
"""
def __init__(self, tid, status, user, desc, ext, pos, stuff):
@@ -2127,9 +2215,8 @@
return r
raise IndexError, index
-
-class Record:
+class Record(BaseStorage.DataRecord):
"""An abstract database record
"""
def __init__(self, *args):
=== StandaloneZODB/ZODB/MappingStorage.py 1.4 => 1.5 ===
old=self._index[oid]
oserial=old[:8]
- if serial != oserial: raise POSException.ConflictError
+ if serial != oserial:
+ raise POSException.ConflictError(serials=(oserial, serial))
serial=self._serial
self._tindex.append((oid,serial+data))
=== StandaloneZODB/ZODB/POSException.py 1.8 => 1.9 ===
#
##############################################################################
-'''BoboPOS-defined exceptions
+"""BoboPOS-defined exceptions
-$Id$'''
-__version__='$Revision$'[11:-2]
+$Id$"""
+__version__ = '$Revision$'.split()[-2:][0]
from string import join
-StringType=type('')
-DictType=type({})
+from types import StringType, DictType
+from ZODB import utils
class POSError(Exception):
"""Persistent object system error
@@ -28,10 +28,94 @@
"""
class ConflictError(TransactionError):
- """Two transactions tried to modify the same object at once
+ """Two transactions tried to modify the same object at once. This
+ transaction should be resubmitted.
- This transaction should be resubmitted.
- """
+ Instance attributes:
+ oid : string
+ the OID (8-byte packed string) of the object in conflict
+ class_name : string
+ the fully-qualified name of that object's class
+ message : string
+ a human-readable explanation of the error
+ serials : (string, string)
+ a pair of 8-byte packed strings; these are the serial numbers
+ (old and new) of the object in conflict. (Serial numbers are
+ closely related [equal?] to transaction IDs; a ConflictError may
+ be triggered by a serial number mismatch.)
+
+ The caller should pass either object or oid as a keyword argument,
+ but not both of them. If object is passed, it should be a
+ persistent object with an _p_oid attribute.
+ """
+
+ def __init__(self, message=None, object=None, oid=None, serials=None):
+ if message is None:
+ self.message = "database conflict error"
+ else:
+ self.message = message
+
+ if object is None:
+ self.oid = None
+ self.class_name = None
+ else:
+ self.oid = object._p_oid
+ klass = object.__class__
+ self.class_name = klass.__module__ + "." + klass.__name__
+
+ if oid is not None:
+ assert self.oid is None
+ self.oid = oid
+
+ self.serials = serials
+
+ def __str__(self):
+ extras = []
+ if self.oid:
+ extras.append("oid %016x" % utils.U64(self.oid))
+ if self.class_name:
+ extras.append("class %s" % self.class_name)
+ if self.serials:
+ extras.append("serial was %016x, now %016x" %
+ tuple(map(utils.U64, self.serials)))
+ if extras:
+ return "%s (%s)" % (self.message, ", ".join(extras))
+ else:
+ return self.message
+
+ def get_oid(self):
+ return self.oid
+
+ def get_class_name(self):
+ return self.class_name
+
+ def get_old_serial(self):
+ return self.serials[0]
+
+ def get_new_serial(self):
+ return self.serials[1]
+
+ def get_serials(self):
+ return self.serials
+
+
+class ReadConflictError(ConflictError):
+ """A conflict detected at read time -- attempt to read an object
+ that has changed in another transaction (eg. another thread
+ or process).
+ """
+ def __init__(self, message=None, object=None, serials=None):
+ if message is None:
+ message = "database read conflict error"
+ ConflictError.__init__(self, message=message, object=object,
+ serials=serials)
+
+class BTreesConflictError(ConflictError):
+ """A special subclass for BTrees conflict errors, which return
+ an undocumented four-tuple."""
+ def __init__(self, *btree_args):
+ ConflictError.__init__(self, message="BTrees conflict error")
+ self.btree = btree_args
class VersionError(POSError):
"""An error in handling versions occurred
@@ -82,6 +166,10 @@
class MountedStorageError(StorageError):
"""Unable to access mounted storage.
+ """
+
+class ReadOnlyError(StorageError):
+ """Unable to modify objects in a read-only storage.
"""
class ExportError(POSError):