[Zope3-checkins] CVS: Zope3/lib/python/ZODB - BaseStorage.py:1.23 ConflictResolution.py:1.15 Connection.py:1.86 DB.py:1.55 ExportImport.py:1.17 FileStorage.py:1.104 MappingStorage.py:1.12 Serialize.py:1.5 fsrecover.py:1.7
Jeremy Hylton
jeremy@zope.com
Tue, 26 Nov 2002 12:41:22 -0500
Update of /cvs-repository/Zope3/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv23161/ZODB
Modified Files:
BaseStorage.py ConflictResolution.py Connection.py DB.py
ExportImport.py FileStorage.py MappingStorage.py Serialize.py
fsrecover.py
Log Message:
Big set of interlocking changes.
Implement much more of pickling functionality in Serialize module.
Rename pickler and unpickler classes to ObjectWriter and ObjectReader,
since they use pickles internally but aren't themselves picklers.
Connection now uses an ObjectWriter instead of calling pickle
directly.
Remove second argument from pack. Call the findrefs() function from
Serialize directly rather than passing it around everywhere. There's
far more work to be done to make serialization pluggable and this only
made matters more complicated.
Change __name__ attribute of storages to _name, since it __ names
suggested that they are reserved or special and this attribute isn't.
XXX Make minimal changes to ExportImport, but it should be using
Serialize rather than using a pickler directly.
Remove new_persistent_id() factory function. persistent_id() is now a
method on the ObjectWriter.
Remove __len__() and getSize() methods from storage API, at least for
now.
=== Zope3/lib/python/ZODB/BaseStorage.py 1.22 => 1.23 ===
--- Zope3/lib/python/ZODB/BaseStorage.py:1.22 Mon Nov 25 14:54:50 2002
+++ Zope3/lib/python/ZODB/BaseStorage.py Tue Nov 26 12:41:21 2002
@@ -228,7 +228,7 @@
def versions(self, max=None):
return ()
- def pack(self, t, referencesf):
+ def pack(self, t):
if self._is_read_only:
raise POSException.ReadOnlyError()
=== Zope3/lib/python/ZODB/ConflictResolution.py 1.14 => 1.15 ===
--- Zope3/lib/python/ZODB/ConflictResolution.py:1.14 Thu Sep 19 15:11:03 2002
+++ Zope3/lib/python/ZODB/ConflictResolution.py Tue Nov 26 12:41:21 2002
@@ -15,7 +15,7 @@
from cPickle import Pickler, PicklingError
from Transaction.Exceptions import ConflictError
-from ZODB.Serialize import ResolveUnpickler, getClassMetadata
+from ZODB.Serialize import ResolveObjectReader, getClassMetadata
ResolvedSerial = "rs"
@@ -53,22 +53,22 @@
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
committedData=None):
- unpickler = ResolveUnpickler(PersistentReferenceFactory())
- resolve = unpickler.getResolver(newpickle)
+ reader = ResolveObjectReader(PersistentReferenceFactory())
+ resolve = reader.getResolver(newpickle)
if resolve is None:
return None
- newstate = unpickler.getState(newpickle)
+ newstate = reader.getState(newpickle)
try:
p = self.loadSerial(oid, oldSerial)
try:
- old = unpickler.getState(p)
+ old = reader.getState(p)
except (EOFError, PicklingError), err:
return None
if committedData is None:
committedData = self.loadSerial(oid, committedSerial)
try:
- committed = unpickler.getState(committedData)
+ committed = reader.getState(committedData)
except (EOFError, PicklingError), err:
return None
resolved = resolve(old, committed, newstate)
=== Zope3/lib/python/ZODB/Connection.py 1.85 => 1.86 ===
--- Zope3/lib/python/ZODB/Connection.py:1.85 Fri Nov 15 16:49:42 2002
+++ Zope3/lib/python/ZODB/Connection.py Tue Nov 26 12:41:21 2002
@@ -48,8 +48,8 @@
from ZODB.ConflictResolution import ResolvedSerial
from ZODB.IConnection import IConnection
from ZODB.POSException import ConflictError, RollbackError
-from ZODB.Serialize import ConnectionUnpickler, \
- new_persistent_id, getClassMetadata
+from ZODB.Serialize import ConnectionObjectReader, \
+ getClassMetadata, ObjectWriter
from ZODB.utils import u64
from Transaction import get_transaction
@@ -77,7 +77,8 @@
multi-threaded application, each thread gets its own object
space.
- The Connection manages movement of objects in and out of object storage.
+ The Connection manages movement of objects in and out of object
+ storage.
"""
# instance variable that holds a TmpStore object used by sub-transactions
@@ -92,13 +93,12 @@
__implements__ = IConnection
def __init__(self, db, version='', cache_size=400):
- """Create a new Connection"""
self._db = db
self._storage = db._storage
self.new_oid = db._storage.new_oid
self._version = version
self._cache = cache = Cache(cache_size)
- self._unpickler = ConnectionUnpickler(self, self._cache)
+ self._reader = ConnectionObjectReader(self, self._cache)
# _invalidated queues invalidate messages delivered from the DB
self._invalidated = Set()
@@ -129,7 +129,7 @@
return object
p, serial = self._storage.load(oid, self._version)
- object = self._unpickler.getGhost(p)
+ object = self._reader.getGhost(p)
object._p_oid = oid
object._p_jar = self
@@ -175,7 +175,7 @@
else:
invalid = 0
- self._unpickler.setGhostState(object, p)
+ self._reader.setGhostState(object, p)
object._p_serial = serial
if invalid:
@@ -290,16 +290,11 @@
else:
return # Nothing to do
- stack = [object]
-
- file = StringIO()
- pickler = cPickle.Pickler(file, 1)
- pickler.persistent_id = new_persistent_id(self, stack)
-
- while stack:
- self.commit_store(stack.pop(), file, pickler, transaction)
+ writer = ObjectWriter(self)
+ for obj in writer.newObjects(object):
+ self.commit_store(writer, obj, transaction)
- def commit_store(self, pobject, file, pickler, transaction):
+ def commit_store(self, writer, pobject, transaction):
# XXX the file and pickler get reset each time through...
# Maybe just create new ones each time? Except I'm not sure
# how that interacts with the persistent_id attribute.
@@ -314,15 +309,8 @@
raise ConflictError(oid=oid)
self._modified.add(oid)
- module, classname, newargs = getClassMetadata(pobject)
- state = pobject.__getstate__()
-
- file.seek(0)
- pickler.clear_memo()
- pickler.dump((module, classname, newargs))
- pickler.dump(state)
- p = file.getvalue(1)
- s = self._storage.store(oid, serial, p, self._version, transaction)
+ s = self._storage.store(oid, serial, writer.getState(pobject),
+ self._version, transaction)
# Put the object in the cache before handling the
# response, just in case the response contains the
# serial number for a newly created object
=== Zope3/lib/python/ZODB/DB.py 1.54 => 1.55 ===
--- Zope3/lib/python/ZODB/DB.py:1.54 Fri Nov 15 16:49:42 2002
+++ Zope3/lib/python/ZODB/DB.py Tue Nov 26 12:41:21 2002
@@ -20,10 +20,9 @@
import POSException
from Connection import Connection
from threading import Lock
-from referencesf import referencesf
from time import time, ctime
from zLOG import LOG, ERROR
-from ZODB.Serialize import Pickler
+from ZODB.Serialize import getDBRoot
from ZODB.ZTransaction import Transaction
from Transaction import get_transaction
@@ -74,12 +73,9 @@
storage.load(ROOT_KEY, "")
except KeyError:
# Create the database's root in the storage if it doesn't exist
- from Persistence.PersistentDict import PersistentDict
- root = PersistentDict()
- p = Pickler()
t = Transaction(description="initial database creation")
storage.tpc_begin(t)
- storage.store(ROOT_KEY, None, p.saveObject(root), '', t)
+ storage.store(ROOT_KEY, None, getDBRoot(), '', t)
storage.tpc_vote(t)
storage.tpc_finish(t)
@@ -292,7 +288,7 @@
t = time()
t -= days * 86400
try:
- self._storage.pack(t,referencesf)
+ self._storage.pack(t)
except:
LOG("ZODB", ERROR, "packing", error=sys.exc_info())
raise
=== Zope3/lib/python/ZODB/ExportImport.py 1.16 => 1.17 ===
--- Zope3/lib/python/ZODB/ExportImport.py:1.16 Wed Jul 24 19:12:46 2002
+++ Zope3/lib/python/ZODB/ExportImport.py Tue Nov 26 12:41:21 2002
@@ -15,7 +15,7 @@
from ZODB import POSException
from ZODB.utils import p64, u64
-from ZODB.referencesf import referencesf
+from ZODB.Serialize import findrefs
from Transaction import get_transaction
from cStringIO import StringIO
@@ -47,7 +47,7 @@
# XXX what exception is expected?
pass # Ick, a broken reference
else:
- referencesf(p, oids)
+ oids += findrefs(p)
file.write(oid)
file.write(p64(len(p)))
file.write(p)
=== Zope3/lib/python/ZODB/FileStorage.py 1.103 => 1.104 ===
--- Zope3/lib/python/ZODB/FileStorage.py:1.103 Tue Nov 26 10:19:46 2002
+++ Zope3/lib/python/ZODB/FileStorage.py Tue Nov 26 12:41:21 2002
@@ -135,6 +135,7 @@
import ZODB.DB
from ZODB import BaseStorage, ConflictResolution, POSException
from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
+from ZODB.Serialize import findrefs
from ZODB.TimeStamp import TimeStamp, newTimeStamp
from ZODB.lock_file import lock_file
from ZODB.utils import p64, u64, cp
@@ -1387,7 +1388,7 @@
file.seek(pos-p+8)
return file.read(1) not in ' u'
- def _pack_index(self, file, index, referencesf):
+ def _pack_index(self, file, index):
rootl = [z64]
pindex = fsIndex()
while rootl:
@@ -1396,18 +1397,19 @@
continue
try:
p, v, nv = self._loada(oid, index, file)
- referencesf(p, rootl)
+ rootl.extend(findrefs(p))
if nv:
p, serial = self._load(oid, '', index, file)
- referencesf(p, rootl)
+ rootl.extend(findrefs(p))
pindex[oid] = index[oid]
except Exception, err:
+ raise
pindex[oid] = 0
error('Bad reference to %s: %s', u64(oid), err)
return pindex
- def pack(self, t, referencesf):
+ def pack(self, t):
"""Copy data from the current database file to a packed file
@@ -1459,7 +1461,7 @@
'The database has already been packed to a later time\n'
'or no changes have been made since the last pack')
- pindex = self._pack_index(file, index, referencesf)
+ pindex = self._pack_index(file, index)
##################################################################
# Step 2, copy data and compute new index based on new positions.
=== Zope3/lib/python/ZODB/MappingStorage.py 1.11 => 1.12 ===
--- Zope3/lib/python/ZODB/MappingStorage.py:1.11 Fri Nov 15 16:49:42 2002
+++ Zope3/lib/python/ZODB/MappingStorage.py Tue Nov 26 12:41:21 2002
@@ -91,6 +91,7 @@
import ZODB.DB
from ZODB import BaseStorage, POSException, utils
+from ZODB.Serialize import findrefs
from ZODB.TimeStamp import TimeStamp
def DB(name="Mapping Storage",
@@ -162,7 +163,7 @@
index=self._index
for oid, p in self._tindex: index[oid]=p
- def pack(self, t, referencesf):
+ def pack(self, t):
self._lock_acquire()
try:
@@ -181,7 +182,7 @@
r=index[oid]
pindex[oid]=r
p=r[8:]
- referencesf(p, rootl)
+ rootl.extend(findrefs(p))
# Now delete any unreferenced entries:
for oid in index.keys():
=== Zope3/lib/python/ZODB/Serialize.py 1.4 => 1.5 ===
--- Zope3/lib/python/ZODB/Serialize.py:1.4 Fri Nov 15 16:49:42 2002
+++ Zope3/lib/python/ZODB/Serialize.py Tue Nov 26 12:41:21 2002
@@ -33,12 +33,20 @@
The class metadata is a three-tuple contained the module name, the
class name, and a tuple of arguments to pass to ``__new__``. The last
element may be None if the only argument to ``__new__`` is the class.
+
+Persistent references
+---------------------
+A persistent reference is a pair containing an oid and class metadata.
+XXX Need to write more about when they are used and when plain oids
+are used.
"""
from cStringIO import StringIO
import cPickle
from types import StringType, TupleType
+import zLOG
+
def getClass(module, name):
mod = __import__(module)
parts = module.split(".")
@@ -56,16 +64,94 @@
newargs = None
return module, classname, newargs
-class Pickler:
+class RootJar:
+ def new_oid(self):
+ return "\0" * 8
+
+def getDBRoot():
+ """Return a serialized database root object."""
+ # Need for the initial bootstrap
+ writer = ObjectWriter(RootJar())
+ from Persistence.PersistentDict import PersistentDict
+ root = PersistentDict()
+ return writer.getState(root)
+
+class ObjectWriter:
+
+ def __init__(self, jar=None):
+ self._file = StringIO()
+ self._p = cPickle.Pickler(self._file, 1)
+ self._p.persistent_id = self._persistent_id
+ self._stack = []
+ if jar is not None:
+ assert hasattr(jar, "new_oid")
+ self._jar = jar
+
+ def _persistent_id(self, obj):
+ """Test if an object is persistent, returning an oid if it is.
+
+ This function is used by the pickler to test whether an object
+ is persistent. If it isn't, the function returns None and the
+ object is included in the pickle for the current persistent
+ object.
+
+ If it is persistent, it returns the oid and sometimes a tuple
+ with other stuff.
+ """
+ if not hasattr(obj, '_p_oid'):
+ return None
+
+ oid = obj._p_oid
+
+ # I'd like to write something like this --
+ # if isinstance(oid, types.MemberDescriptor):
+ # -- but I can't because the type doesn't have a canonical name.
+ # Instead, we'll assert that an oid must always be a string
+ if not (oid is None or isinstance(oid, StringType)):
+ # XXX log a warning
+ return None
- def saveObject(self, object):
- file = StringIO()
- p = cPickle.Pickler(file, 1)
- p.dump(getClassMetadata(object))
- p.dump(object.__getstate__())
- return file.getvalue()
+ if oid is None or obj._p_jar is not self._jar:
+ # XXX log warning if obj._p_jar is not self
+ oid = self._jar.new_oid()
+ obj._p_jar = self._jar
+ obj._p_oid = oid
+ self._stack.append(obj)
+
+ return oid, getClassMetadata(obj)
+
+ def newObjects(self, obj):
+ # The modified object is also a "new" object.
+ # XXX Should only call newObjects() once per Pickler.
+ self._stack.append(obj)
+ return NewObjectIterator(self._stack)
+
+ def getState(self, obj):
+ self._file.reset()
+ self._p.clear_memo()
+ self._p.dump(getClassMetadata(obj))
+ self._p.dump(obj.__getstate__())
+ return self._file.getvalue()
+
+class NewObjectIterator:
+
+ # The pickler is used as a forward iterator when the connection
+ # is looking for new objects to pickle.
+
+ def __init__(self, stack):
+ self._stack = stack
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ if self._stack:
+ elt = self._stack.pop()
+ return elt
+ else:
+ raise StopIteration
-class BaseUnpickler:
+class BaseObjectReader:
# subclasses must define _persistent_load().
@@ -107,14 +193,14 @@
object.__setstate__(state)
return object
-class ConnectionUnpickler(BaseUnpickler):
+class ConnectionObjectReader(BaseObjectReader):
def __init__(self, conn, cache):
self._conn = conn
self._cache = cache
def _persistent_load(self, oid):
- # persistent_load function to pass to Unpickler
+ # persistent_load function to pass to ObjectReader
if isinstance(oid, TupleType):
# XXX We get here via new_persistent_id()
@@ -140,7 +226,7 @@
return object
return self._conn[oid]
-class ResolveUnpickler(BaseUnpickler):
+class ResolveObjectReader(BaseObjectReader):
bad_classes = {}
@@ -168,43 +254,20 @@
else:
return resolve
-def new_persistent_id(self, stack):
- # XXX need a doc string. not sure if the one for persistent_id()
- # below is correct.
-
- # Create a special persistent_id that captures T and the subobject
- # stack in a closure.
-
- def persistent_id(object):
- """Test if an object is persistent, returning an oid if it is.
-
- This function is used by the pickler to test whether an object
- is persistent. If it isn't, the function returns None and the
- object is included in the pickle for the current persistent
- object.
-
- If it is persistent, it returns the oid and sometimes a tuple
- with other stuff.
- """
- if not hasattr(object, '_p_oid'):
- return None
-
- oid = object._p_oid
-
- # I'd like to write something like this --
- # if isinstance(oid, types.MemberDescriptor):
- # -- but I can't because the type doesn't have a canonical name.
- # Instead, we'll assert that an oid must always be a string
- if not (isinstance(oid, StringType) or oid is None):
- return None
-
- if oid is None or object._p_jar is not self:
- oid = self.new_oid()
- object._p_jar = self
- object._p_oid = oid
- stack.append(object)
-
- return oid, getClassMetadata(object)
-
- return persistent_id
-
+def findrefs(p):
+ f = StringIO(p)
+ u = cPickle.Unpickler(f)
+ u.persistent_load = L = []
+ u.noload()
+ try:
+ u.noload()
+ except EOFError, err:
+ zLOG.LOG("ZODB", zLOG.INFO, "Bad pickled: %s" % err)
+ # Iterator over L and convert persistent references to simple oids.
+ oids = []
+ for ref in L:
+ if isinstance(ref, TupleType):
+ oids.append(ref[0])
+ else:
+ oids.append(ref)
+ return oids
=== Zope3/lib/python/ZODB/fsrecover.py 1.6 => 1.7 ===
--- Zope3/lib/python/ZODB/fsrecover.py:1.6 Thu Aug 1 12:24:21 2002
+++ Zope3/lib/python/ZODB/fsrecover.py Tue Nov 26 12:41:21 2002
@@ -318,8 +318,7 @@
if pack is not None:
print "Packing ..."
- from ZODB.referencesf import referencesf
- ofs.pack(pack, referencesf)
+ ofs.pack(pack)
ofs.close()