[Zodb-checkins] SVN: ZODB/branches/ctheune-betterimport/
Restructured the oid mapping when importing ZEXP files and
added support for
Christian Theune
ct at gocept.com
Thu Aug 16 07:37:54 EDT 2007
Log message for revision 78876:
Restructured the oid mapping when importing ZEXP files and added support for
importing weakrefs.
Changed:
U ZODB/branches/ctheune-betterimport/NEWS.txt
U ZODB/branches/ctheune-betterimport/src/ZODB/ExportImport.py
U ZODB/branches/ctheune-betterimport/src/ZODB/blob.py
U ZODB/branches/ctheune-betterimport/src/ZODB/tests/testZODB.py
-=-
Modified: ZODB/branches/ctheune-betterimport/NEWS.txt
===================================================================
--- ZODB/branches/ctheune-betterimport/NEWS.txt 2007-08-16 11:35:34 UTC (rev 78875)
+++ ZODB/branches/ctheune-betterimport/NEWS.txt 2007-08-16 11:37:53 UTC (rev 78876)
@@ -15,6 +15,9 @@
- (3.9.0a1) Make it possible to examine oid and (in some situations) database
name of persistent object references during conflict resolution.
+- (3.9.0b1) Restructured the ZEXP import code and added support for importing
+ weak references.
+
ZEO
---
Modified: ZODB/branches/ctheune-betterimport/src/ZODB/ExportImport.py
===================================================================
--- ZODB/branches/ctheune-betterimport/src/ZODB/ExportImport.py 2007-08-16 11:35:34 UTC (rev 78875)
+++ ZODB/branches/ctheune-betterimport/src/ZODB/ExportImport.py 2007-08-16 11:37:53 UTC (rev 78876)
@@ -20,6 +20,8 @@
from tempfile import TemporaryFile
import logging
+import zope.interface
+
from ZODB.blob import Blob
from ZODB.interfaces import IBlobStorage
from ZODB.POSException import ExportError, POSKeyError
@@ -103,30 +105,71 @@
Appends one item, the OID of the first object created,
to return_oid_list.
"""
+
+ # A mapping from an oid in the import to a new oid in the storage.
oids = {}
- # IMPORTANT: This code should be consistent with the code in
- # serialize.py. It is currently out of date and doesn't handle
- # weak references.
+ def persistent_load(old_reference):
+ """Remap a persistent reference to a new oid and create a ghost for it.
- def persistent_load(ooid):
- """Remap a persistent id to a new ID and create a ghost for it."""
+ The remap has to take care for the various cases of persistent references
+ as described in serialize.py.
- klass = None
- if isinstance(ooid, tuple):
- ooid, klass = ooid
+ """
+ # IMPORTANT: This code should be consistent with the code in
+ # serialize.py.
- if ooid in oids:
- oid = oids[ooid]
- else:
- if klass is None:
- oid = self._storage.new_oid()
+ # Notice: The mapping `oids` is held in the parent namespace of this
+ # function to make the signature of this function compatible with
+ # the unpickler.
+ if isinstance(old_reference, str):
+ # Case 1: A simple object reference
+ # Format: oid
+ reference = SimpleObjectReference(old_reference)
+
+ elif isinstance(old_reference, tuple) and len(old_reference) == 2:
+ # Case 2: A persistent object reference
+ # Format: (oid, class meta data)
+ reference = PersistentObjectReference(*old_reference)
+
+ elif isinstance(old_reference, list) and len(old_reference) > 1:
+ # Case 3: An extended reference
+ # Format: [reference_type, args]
+ extension_type, args = old_reference
+
+ if extension_type == 'w':
+ # Case 3a: A weak reference
+ # Format: ['w', oid]
+ reference = PersistentWeakReference(args[0])
+
+ elif extension_type == 'n':
+ # Case 3b: a multi-database simple object reference ['n', db_name,
+ # oid] (XXX, currently unsupported)
+ raise TypeError('Importing simple multi-database object '
+ 'references is not supported.')
+ elif extension_type == 'm':
+ # Case 3c: a multi-database persistent object reference ['m',
+ # dbname, oid, metadata] (XXX, currently unsupported)
+ raise TypeError('Importing persistent multi-database object '
+ 'references is not supported.')
+
else:
- oid = self._storage.new_oid(), klass
- oids[ooid] = oid
+ raise TypeError('Unknown extended reference type: %r' % extension_type)
- return Ghost(oid)
+ elif isinstance(old_reference, list) and isinstance(str, old_reference):
+ # Case 4: A supported legacy format of a weak reference:
+ # [oid]
+ reference = PersistentWeakReference(old_reference[0])
+ else:
+ raise Exception('Invalid persistent reference format: %r' % old_reference)
+
+ # Ask the reference to remap any oids from the old storage.
+ reference.map(self._storage, oids)
+ # Create a reference representation again and return the
+ # corresponding ghost.
+ return Ghost(reference.serialize())
+
version = self._version
while 1:
@@ -146,8 +189,6 @@
if oids:
oid = oids[ooid]
- if isinstance(oid, tuple):
- oid = oid[0]
else:
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
@@ -196,3 +237,79 @@
def persistent_id(obj):
if isinstance(obj, Ghost):
return obj.oid
+
+
+class IPersistentReference(zope.interface.Interface):
+
+ oid = zope.interface.Attribute("The oid that is referenced")
+
+ def serialize():
+ """Returns a serialized version of the reference, compatible with the
+ structures from serialize.py"""
+
+
+class ReferenceBase(object):
+ """Semi-abstract base class for references."""
+
+ zope.interface.implements(IPersistentReference)
+
+ # A helper to avoid accidental double mapping.
+ mapped = False
+
+ def map(self, storage, oids):
+ """Actually map the oid and store the mapping for future use.
+
+ If the oid was already mapped before, the re-use the mapped oid.
+
+ """
+ assert not self.mapped, "Trying to map reference multiple times."
+
+ if self.oid not in oids:
+ # The oid wasn't mapped yet, so map it.
+ oids[self.oid] = storage.new_oid()
+
+ self.oid = oids[self.oid]
+ self.mapped = True
+
+
+class SimpleObjectReference(ReferenceBase):
+ """A simple object reference.
+
+ Serialization format: oid
+
+ """
+
+ def __init__(self, oid):
+ self.oid = oid
+
+ def serialize(self):
+ return oid
+
+
+class PersistentObjectReference(ReferenceBase):
+ """A persistent object reference.
+
+ Serialization format: (oid, class meta data)
+
+ """
+
+ def __init__(self, oid, metadata):
+ self.oid = oid
+ self.metadata = metadata
+
+ def serialize(self):
+ return (self.oid, self.metadata)
+
+
+class PersistentWeakReference(ReferenceBase):
+ """A persistent weak reference
+
+ Serialization format (extended): ['w', (oid,)]
+
+ """
+
+ def __init__(self, oid):
+ self.oid = oid
+
+ def serialize(self):
+ return ['w', (self.oid,)]
Modified: ZODB/branches/ctheune-betterimport/src/ZODB/blob.py
===================================================================
--- ZODB/branches/ctheune-betterimport/src/ZODB/blob.py 2007-08-16 11:35:34 UTC (rev 78875)
+++ ZODB/branches/ctheune-betterimport/src/ZODB/blob.py 2007-08-16 11:37:53 UTC (rev 78876)
@@ -447,7 +447,8 @@
while self.dirty_oids:
oid, serial = self.dirty_oids.pop()
clean = self.fshelper.getBlobFilename(oid, serial)
- if os.exists(clean):
+ # XXX This code is untested.
+ if os.path.exists(clean):
remove_committed(clean)
@non_overridable
Modified: ZODB/branches/ctheune-betterimport/src/ZODB/tests/testZODB.py
===================================================================
--- ZODB/branches/ctheune-betterimport/src/ZODB/tests/testZODB.py 2007-08-16 11:35:34 UTC (rev 78875)
+++ ZODB/branches/ctheune-betterimport/src/ZODB/tests/testZODB.py 2007-08-16 11:37:53 UTC (rev 78876)
@@ -21,7 +21,7 @@
from ZODB.POSException import TransactionFailedError
from ZODB.tests.warnhook import WarningsHook
-from persistent import Persistent
+from persistent import Persistent, wref
from persistent.mapping import PersistentMapping
import transaction
@@ -78,6 +78,36 @@
finally:
conn.close()
+ def checkExportImport_weakref(self):
+ conn = self._db.open()
+ root = conn.root()
+ # Fill the database
+ transaction.begin()
+
+ # Set up an isolated area with a target and a weakref.
+ parent = root['parent'] = PersistentMapping()
+ parent['target'] = PersistentMapping()
+ parent['weakref'] = wref.WeakRef(parent['target'])
+ transaction.commit()
+
+ # Export that area.
+ transaction.begin()
+ import tempfile
+ f = tempfile.TemporaryFile()
+ parent._p_jar.exportFile(parent._p_oid, f)
+
+ # Import it again
+ f.seek(0)
+ root['parent2'] = root._p_jar.importFile(f)
+ transaction.commit()
+
+ # Verify that the weakref points to the parent form the import, not
+ # the original target.
+ # same target
+ self.failUnless(root['parent2']['weakref']() is
+ root['parent2']['target'])
+
+
def duplicate(self, conn, abort_it):
transaction.begin()
transaction.get().note('duplication')
More information about the Zodb-checkins
mailing list