[Zope3-checkins] CVS: Zope3/lib/python/ZODB - BaseStorage.py:1.22 FileStorage.py:1.102 dbmStorage.py:1.5 utils.py:1.14
Jeremy Hylton
jeremy@zope.com
Mon, 25 Nov 2002 14:54:51 -0500
Update of /cvs-repository/Zope3/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv7770/ZODB
Modified Files:
BaseStorage.py FileStorage.py dbmStorage.py utils.py
Log Message:
First step in ZODB3 / ZODB4 integration.
Merge FileStorage and BaseStorage changes and all the other dependent
stuff.
Make copyTransactionsFrom() always use restore().
Replace all uses of U64 with u64.
Use newTimeStamp() from ZODB.TimeStamp.
Remove __len__() and getSize().
Add getExtensionMethods().
=== Zope3/lib/python/ZODB/BaseStorage.py 1.21 => 1.22 ===
--- Zope3/lib/python/ZODB/BaseStorage.py:1.21 Wed Jul 24 19:12:46 2002
+++ Zope3/lib/python/ZODB/BaseStorage.py Mon Nov 25 14:54:50 2002
@@ -17,36 +17,33 @@
"""
import threading
-import time
-import POSException
-from TimeStamp import TimeStamp
+from ZODB import POSException
+from ZODB.TimeStamp import newTimeStamp, TimeStamp
z64='\0'*8
class BaseStorage:
- _transaction=None # Transaction that is being committed
- _serial=z64 # Transaction serial number
- _tstatus=' ' # Transaction status, used for copying data
- _is_read_only = 0
+ _transaction = None # Transaction that is being committed
+ _serial = z64 # Transaction serial number
+ _tstatus = ' ' # Transaction status, used for copying data
+ _is_read_only = False
def __init__(self, name, base=None):
-
- self.__name__=name
+ self._name = name
# Allocate locks:
l=threading.RLock()
- self._lock_acquire=l.acquire
- self._lock_release=l.release
+ self._lock_acquire = l.acquire
+ self._lock_release = l.release
l=threading.Lock()
- self._commit_lock_acquire=l.acquire
- self._commit_lock_release=l.release
+ self._commit_lock_acquire = l.acquire
+ self._commit_lock_release = l.release
- t=time.time()
- t=self._ts=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
- self._serial=t.raw()
+ self._ts = newTimeStamp()
+ self._serial = self._ts.raw()
if base is None:
- self._oid='\0\0\0\0\0\0\0\0'
+ self._oid = z64
else:
- self._oid=base._oid
+ self._oid = base._oid
def abortVersion(self, src, transaction):
if transaction is not self._transaction:
@@ -61,11 +58,14 @@
def close(self):
pass
+ def sortKey(self):
+ # An implementation should override this if it can be shared
+ # by multiple processes and / or guarantee that self._name is
+ # uniquely defines the storage across all procsses.
+ return self._name
+
def getName(self):
- return self.__name__
-
- def getSize(self):
- return len(self)*300 # WAG!
+ return self._name
def history(self, oid, version, length=1):
pass
@@ -74,6 +74,7 @@
return ''
def new_oid(self, last=None):
+ # 'last' is only for internal use, not part of the public API
if self._is_read_only:
raise POSException.ReadOnlyError()
if last is None:
@@ -98,7 +99,7 @@
return self._is_read_only
def supportsVersions(self):
- return 0
+ return False
def tpc_abort(self, transaction):
self._lock_acquire()
@@ -106,15 +107,18 @@
if transaction is not self._transaction: return
self._abort()
self._clear_temp()
- self._transaction=None
+ self._transaction = None
self._commit_lock_release()
- finally: self._lock_release()
+ finally:
+ self._lock_release()
def _abort(self):
- """Subclasses should redefine this to supply abort actions"""
+ # Subclasses should define this to supply abort actions.
pass
def tpc_begin(self, transaction, tid=None, status=' '):
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
@@ -122,51 +126,41 @@
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
- self._transaction=transaction
+ self._transaction = transaction
self._clear_temp()
- user=transaction.user
- desc=transaction.description
- ext=transaction._extension
- if ext:
+ if transaction._extension:
import cPickle
- ext = cPickle.dumps(ext, 1)
+ ext = cPickle.dumps(transaction._extension, 1)
else:
ext = ""
- self._ude=user, desc, ext
-
+ self._ude= transaction.user, transaction.description, ext
if tid is None:
- t=time.time()
- t=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
- self._ts=t=t.laterThan(self._ts)
- self._serial=t.raw()
+ self._ts = newTimeStamp(self._ts)
+ self._serial = self._ts.raw()
else:
- self._ts=TimeStamp(tid)
- self._serial=tid
-
- self._tstatus=status
-
- self._begin(self._serial, user, desc, ext)
-
+ self._ts = TimeStamp(tid)
+ self._serial = tid
+ self._tstatus = status
+ self._begin(self._serial, *self._ude)
finally:
self._lock_release()
def _begin(self, tid, u, d, e):
- """Subclasses should redefine this to supply transaction start actions.
- """
+ # Subclasses should define this to supply transaction start actions.
pass
def tpc_vote(self, transaction):
self._lock_acquire()
try:
- if transaction is not self._transaction: return
+ if transaction is not self._transaction:
+ return
self._vote()
finally:
self._lock_release()
def _vote(self):
- """Subclasses should redefine this to supply transaction vote actions.
- """
+ # Subclasses should define this to supply transaction vote actions.
pass
def tpc_finish(self, transaction, f=None):
@@ -177,9 +171,7 @@
try:
if f is not None:
f()
-
- u, d, e = self._ude
- self._finish(self._serial, u, d, e)
+ self._finish(self._serial, *self._ude)
self._clear_temp()
finally:
self._ude = None
@@ -189,10 +181,11 @@
self._lock_release()
def _finish(self, tid, u, d, e):
- """Subclasses should redefine this to supply transaction finish actions
- """
+ # Subclasses should define this to supply transaction finish actions.
pass
+ # XXX One of these two should go
+
def undoInfo(self, first=0, last=-20, specification=None):
"""Return a list of transaction descriptions for use with undo.
@@ -230,7 +223,7 @@
return ()
def versionEmpty(self, version):
- return 1
+ return True
def versions(self, max=None):
return ()
@@ -252,67 +245,56 @@
raise POSException.Unsupported, (
"Retrieval of historical revisions is not supported")
+ def getExtensionMethods(self):
+ """getExtensionMethods
+
+ This returns a dictionary whose keys are names of extra methods
+ provided by this storage. Storage proxies (such as ZEO) should
+ call this method to determine the extra methods that they need
+ to proxy in addition to the standard storage methods.
+ Dictionary values should be None; this will be a handy place
+ for extra marshalling information, should we need it
+ """
+ return {}
+
def copyTransactionsFrom(self, other, verbose=0):
"""Copy transactions from another storage.
- This is typically used for converting data from one storage to another.
+ This is typically used for converting data from one storage to
+ another.
"""
- _ts=None
- ok=1
- preindex={};
- preget=preindex.get # waaaa
- # restore() is a new storage API method which has an identical
- # signature to store() except that it does not return anything.
- # Semantically, restore() is also identical to store() except that it
- # doesn't do the ConflictError or VersionLockError consistency
- # checks. The reason to use restore() over store() in this method is
- # that store() cannot be used to copy transactions spanning a version
- # commit or abort, or over transactional undos.
- #
- # We'll use restore() if it's available, otherwise we'll fall back to
- # using store(). However, if we use store, then
- # copyTransactionsFrom() may fail with VersionLockError or
- # ConflictError.
- if hasattr(self, 'restore'):
- restoring = 1
- else:
- restoring = 0
+ _ts = None
+ ok = True
for transaction in other.iterator():
-
- tid=transaction.tid
+ tid = transaction.tid
if _ts is None:
- _ts=TimeStamp(tid)
+ _ts = TimeStamp(tid)
else:
- t=TimeStamp(tid)
+ t = TimeStamp(tid)
if t <= _ts:
- if ok: print ('Time stamps out of order %s, %s' % (_ts, t))
- ok=0
- _ts=t.laterThan(_ts)
- tid=_ts.raw()
+ if ok:
+ print ('Time stamps out of order %s, %s' % (_ts, t))
+ ok = False
+ _ts = t.laterThan(_ts)
+ tid = _ts.raw()
else:
_ts = t
if not ok:
print ('Time stamps back in order %s' % (t))
- ok=1
+ ok = True
if verbose: print _ts
self.tpc_begin(transaction, tid, transaction.status)
for r in transaction:
- oid=r.oid
- if verbose: print `oid`, r.version, len(r.data)
- if restoring:
- self.restore(oid, r.serial, r.data, r.version, transaction)
- else:
- pre=preget(oid, None)
- s=self.store(oid, pre, r.data, r.version, transaction)
- preindex[oid]=s
-
+ if verbose: print `r.oid`, r.version, len(r.data)
+ self.restore(r.oid, r.serial, r.data, r.version,
+ r.data_txn, transaction)
self.tpc_vote(transaction)
self.tpc_finish(transaction)
class TransactionRecord:
- """Abstract base class for iterator protocol"""
+ """Abstract base class for iterator protocol."""
class DataRecord:
- """Abstract base class for iterator protocol"""
+ """Abstract base class for iterator protocol."""
=== Zope3/lib/python/ZODB/FileStorage.py 1.101 => 1.102 === (1836/1936 lines abridged)
--- Zope3/lib/python/ZODB/FileStorage.py:1.101 Fri Nov 15 16:49:42 2002
+++ Zope3/lib/python/ZODB/FileStorage.py Mon Nov 25 14:54:50 2002
@@ -2,52 +2,52 @@
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
-#
+#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
-#
+#
##############################################################################
"""File-based ZODB storage
-
+
Files are arranged as follows.
- The first 4 bytes are a file identifier.
-
+
- The rest of the file consists of a sequence of transaction
"records".
A transaction record consists of:
- 8-byte transaction id, which is also a time stamp.
-
+
- 8-byte transaction record length - 8.
-
+
- 1-byte status code
-
+
- 2-byte length of user name
-
- - 2-byte length of description
-
- - 2-byte length of extension attributes
-
+
+ - 2-byte length of description
+
+ - 2-byte length of extension attributes
+
[-=- -=- -=- 1836 lines omitted -=- -=- -=-]
+ self.packt = packt
+ self.first = first
+ self.last = last
+ self.filter = filter
+ self.i = 0
+ self.results = []
+ self.stop = 0
+
+ def finished(self):
+ """Return True if UndoSearch has found enough records."""
+ # BAW: Why 39 please? This makes no sense (see also below).
+ return self.i >= self.last or self.pos < 39 or self.stop
+
+ def search(self):
+ """Search for another record."""
+ dict = self._readnext()
+ if dict is not None and (self.filter is None or self.filter(dict)):
+ if self.i >= self.first:
+ self.results.append(dict)
+ self.i += 1
+
+ def _readnext(self):
+ """Read the next record from the storage."""
+ self.file.seek(self.pos - 8)
+ self.pos -= u64(self.file.read(8)) + 8
+ self.file.seek(self.pos)
+ h = self.file.read(TRANS_HDR_LEN)
+ tid, tl, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+ if tid < self.packt or status == 'p':
+ self.stop = 1
+ return None
+ if status != ' ':
+ return None
+ d = u = ''
+ if ul:
+ u = self.file.read(ul)
+ if dl:
+ d = self.file.read(dl)
+ e = {}
+ if el:
+ try:
+ e = loads(self.file.read(el))
+ except:
+ pass
+ d = {'id': base64.encodestring(tid).rstrip(),
+ 'time': TimeStamp(tid).timeTime(),
+ 'user_name': u,
+ 'description': d}
+ d.update(e)
+ return d
=== Zope3/lib/python/ZODB/dbmStorage.py 1.4 => 1.5 ===
--- Zope3/lib/python/ZODB/dbmStorage.py:1.4 Mon Jun 10 19:27:44 2002
+++ Zope3/lib/python/ZODB/dbmStorage.py Mon Nov 25 14:54:50 2002
@@ -43,11 +43,11 @@
self._lock_acquire()
try:
try:
- return (os.stat(self.__name__+'.data')[6] +
- os.stat(self.__name__+'.dir')[6]
+ return (os.stat(self._name+'.data')[6] +
+ os.stat(self._name+'.dir')[6]
)
except:
- try: return os.stat(self.__name__)[6]
+ try: return os.stat(self._name)[6]
except: return 0
finally: self._lock_release()
@@ -70,7 +70,7 @@
def getSize(self):
self._lock_acquire()
- try: return os.stat(self.__name__)[6]
+ try: return os.stat(self._name)[6]
finally: self._lock_release()
def pack(self, t, referencesf):
=== Zope3/lib/python/ZODB/utils.py 1.13 => 1.14 ===
--- Zope3/lib/python/ZODB/utils.py:1.13 Thu Aug 1 12:11:04 2002
+++ Zope3/lib/python/ZODB/utils.py Mon Nov 25 14:54:50 2002
@@ -19,18 +19,10 @@
"""Pack an integer or long into a 8-byte string"""
return struct.pack(">Q", v)
-# Note that the distinction between ints and longs is blurred in
-# Python 2.2. So make u64() and U64() the same.
-
def u64(v):
"""Unpack an 8-byte string into a 64-bit long integer."""
return struct.unpack(">Q", v)[0]
-U64 = u64
-
-# clients use this module to get this contant
-t32 = 1L << 32
-
def cp(f1, f2, l):
read = f1.read
write = f2.write
@@ -44,11 +36,3 @@
break
write(d)
l = l - len(d)
-
-
-def newTimeStamp(old=None):
- t = time.time()
- ts = TimeStamp(time.gmtime(t)[:5]+(t%60,))
- if old is not None:
- return ts.laterThan(old)
- return ts