[Checkins] SVN: Sandbox/shane/ Split the serialization format patch into ZODB 3.8 and trunk versions.
Shane Hathaway
shane at hathawaymix.org
Thu Jan 15 02:56:41 EST 2009
Log message for revision 94739:
Split the serialization format patch into ZODB 3.8 and trunk versions.
Changed:
A Sandbox/shane/serialization-format-ZODB-3-8.patch
A Sandbox/shane/serialization-format-ZODB-trunk.patch
D Sandbox/shane/zodb-serialization-format.patch
-=-
Copied: Sandbox/shane/serialization-format-ZODB-3-8.patch (from rev 94738, Sandbox/shane/zodb-serialization-format.patch)
===================================================================
--- Sandbox/shane/serialization-format-ZODB-3-8.patch (rev 0)
+++ Sandbox/shane/serialization-format-ZODB-3-8.patch 2009-01-15 07:56:31 UTC (rev 94739)
@@ -0,0 +1,664 @@
+Index: src/ZODB/interfaces.py
+===================================================================
+--- src/ZODB/interfaces.py (revision 93829)
++++ src/ZODB/interfaces.py (working copy)
+@@ -955,3 +1005,87 @@
+
+ class BlobError(Exception):
+ pass
++
++class ISerialFormat(Interface):
++ """A method of serializing objects."""
++
++ def makeSerializer(persistent_id):
++ """Returns a new ISerializer provider.
++
++ The ISerializer must use the given persistent_id function.
++ """
++
++ def makeDeserializer(persistent_load, find_global=None):
++ """Returns a new IDeserializer provider.
++
++ The IDeserializer must use the given persistent_load and find_global
++ functions. The find_global parameter is optional.
++ """
++
++ def listPersistentReferences(data):
++ """Return the list of persistent references from a serialized object.
++
++ If a format name header was provided when the object was dumped,
++ that header will still be in the data parameter.
++
++ Each reference in the returned list may take any of the forms
++ described in the "Persistent References" section of the ZODB.serialize
++ module documentation, such as a simple oid string,
++ (oid, class meta data), or [reference_type, args].
++ """
++
++class ISerializer(Interface):
++ """Serializes objects."""
++
++ def dump(classmeta, state):
++ """Return serialized data given class metadata and an instance state.
++
++ The implementation of this method must use the persistent_id
++ function (provided by makeDumper()) when storing persistent
++ references. The persistent_id method both provides an identity
++ for targets and provides a way for the caller to generate a list of
++ persistent references.
++
++ The implementation needs to prepend a serialization format name
++ (in curly braces) unless this format is the default format.
++ """
++
++class IDeserializer(Interface):
++ """Deserializes objects."""
++
++ def getClassAndState(data):
++ """Return an iterator that produces the class metadata, then the state.
++
++ The recommended implementation is to use two yield statements,
++ making the method a generator.
++ """
++
++ def getClassMetadata(data):
++ """Return the class metadata portion of a serialized object.
++
++ If a format name header was provided when the object was dumped,
++ that header will still be in the data parameter.
++
++ The returned value can be any type of class metadata,
++ such as a class object or a tuple of (class, class_args);
++ the first element of that tuple can be a
++ (module_name, class_name) tuple instead of a class.
++
++ Implementations must use the persistent_load and find_global
++ functions provided to makeDeserializer() when they need to
++ resolve persistent references.
++ """
++
++ def getState(data):
++ """Return the state portion of a serialized object.
++
++ If a format name header was provided when the object was dumped,
++ that header will still be in the data parameter.
++
++ At some point, the returned state will be passed, unchanged,
++ to the __setstate__ method of an existing empty object.
++
++ Implementations must use the persistent_load and find_global
++ functions provided to makeDeserializer() when they need to
++ resolve persistent references.
++ """
+Index: src/ZODB/serialize.py
+===================================================================
+--- src/ZODB/serialize.py (revision 93829)
++++ src/ZODB/serialize.py (working copy)
+@@ -13,8 +13,9 @@
+ ##############################################################################
+ """Support for ZODB object serialization.
+
+-ZODB serializes objects using a custom format based on Python pickles.
+-When an object is unserialized, it can be loaded as either a ghost or
++Multiple types of object serialization are possible. The default
++serialization uses a custom format based on Python pickles.
++When an object is deserialized, it can be loaded as either a ghost or
+ a real object. A ghost is a persistent object of the appropriate type
+ but without any state. The first time a ghost is accessed, the
+ persistence machinery traps access and loads the actual state. A
+@@ -24,8 +25,8 @@
+ Pickle format
+ -------------
+
+-ZODB stores serialized objects using a custom format based on pickle.
+-Each serialized object has two parts: the class description and the
++By default, ZODB stores serialized objects using a custom format based on
++pickle. Each serialized object has two parts: the class description and the
+ object state. The class description must provide enough information
+ to call the class's ``__new__`` and create an empty object. Once the
+ object exists as a ghost, its state is passed to ``__setstate__``.
+@@ -133,16 +134,14 @@
+
+ """
+
+-import cPickle
+-import cStringIO
+ import logging
+
+-
+ from persistent import Persistent
+ from persistent.wref import WeakRefMarker, WeakRef
+ from ZODB import broken
+ from ZODB.broken import Broken
+ from ZODB.POSException import InvalidObjectReference
++from ZODB.format import detect_format, get_format
+
+ _oidtypes = str, type(None)
+
+@@ -163,17 +162,15 @@
+ class ObjectWriter:
+ """Serializes objects for storage in the database.
+
+- The ObjectWriter creates object pickles in the ZODB format. It
+- also detects new persistent objects reachable from the current
+- object.
++ Objects can specify what serialization format to use, but the
++ default format is the ZODB pickle format. This class also detects
++ new persistent objects reachable from the current object.
+ """
+
+ _jar = None
+
+ def __init__(self, obj=None):
+- self._file = cStringIO.StringIO()
+- self._p = cPickle.Pickler(self._file, 1)
+- self._p.persistent_id = self.persistent_id
++ self._dumpers = {} # {serial format name -> dump method}
+ self._stack = []
+ if obj is not None:
+ self._stack.append(obj)
+@@ -268,16 +265,16 @@
+
+ # Most objects are not persistent. The following cheap test
+ # identifies most of them. For these, we return None,
+- # signalling that the object should be pickled normally.
++ # signalling that the object should be serialized normally.
+ if not isinstance(obj, (Persistent, type, WeakRef)):
+- # Not persistent, pickle normally
++ # Not persistent, serialize normally
+ return None
+
+ # Any persistent object must have an oid:
+ try:
+ oid = obj._p_oid
+ except AttributeError:
+- # Not persistent, pickle normally
++ # Not persistent, serialize normally
+ return None
+
+ if not (oid is None or isinstance(oid, str)):
+@@ -287,7 +284,7 @@
+ if hasattr(oid, '__get__'):
+ # The oid is a descriptor. That means obj is a non-persistent
+ # class whose instances are persistent, so ...
+- # Not persistent, pickle normally
++ # Not persistent, serialize normally
+ return None
+
+ if oid is WeakRefMarker:
+@@ -391,39 +388,36 @@
+ if (isinstance(getattr(klass, '_p_oid', 0), _oidtypes)
+ and klass.__module__):
+ # This is a persistent class with a non-empty module. This
+- # uses pickle format #3 or #7.
++ # uses class metadata format #3 or #7.
+ klass = klass.__module__, klass.__name__
+ if newargs is None:
+ meta = klass, None
+ else:
+ meta = klass, newargs()
+ elif newargs is None:
+- # Pickle format #1.
++ # Class metadata format #1.
+ meta = klass
+ else:
+- # Pickle format #2.
++ # Class metadata format #2.
+ meta = klass, newargs()
+
+- return self._dump(meta, obj.__getstate__())
++ state = obj.__getstate__()
+
+- def _dump(self, classmeta, state):
+- # To reuse the existing cStringIO object, we must reset
+- # the file position to 0 and truncate the file after the
+- # new pickle is written.
+- self._file.seek(0)
+- self._p.clear_memo()
+- self._p.dump(classmeta)
+- self._p.dump(state)
+- self._file.truncate()
+- return self._file.getvalue()
++ serial_format_name = getattr(state, 'serial_format', '')
++ dump = self._dumpers.get(serial_format_name)
++ if dump is None:
++ sf = get_format(serial_format_name)
++ dump = sf.makeSerializer(self.persistent_id).dump
++ self._dumpers[serial_format_name] = dump
++ return dump(meta, state)
+
+ def __iter__(self):
+ return NewObjectIterator(self._stack)
+
+ class NewObjectIterator:
+
+- # The pickler is used as a forward iterator when the connection
+- # is looking for new objects to pickle.
++ # The ObjectWriter is used as a forward iterator when the connection
++ # is looking for new objects to serialize.
+
+ def __init__(self, stack):
+ self._stack = stack
+@@ -448,20 +442,16 @@
+ def _get_class(self, module, name):
+ return self._factory(self._conn, module, name)
+
+- def _get_unpickler(self, pickle):
+- file = cStringIO.StringIO(pickle)
+- unpickler = cPickle.Unpickler(file)
+- unpickler.persistent_load = self._persistent_load
++ def _get_deserializer(self, p):
+ factory = self._factory
+ conn = self._conn
+
+ def find_global(modulename, name):
+ return factory(conn, modulename, name)
+
+- unpickler.find_global = find_global
++ sf = detect_format(p)
++ return sf.makeDeserializer(self._persistent_load, find_global)
+
+- return unpickler
+-
+ loaders = {}
+
+ def _persistent_load(self, reference):
+@@ -554,9 +544,9 @@
+
+ return obj
+
+- def getClassName(self, pickle):
+- unpickler = self._get_unpickler(pickle)
+- klass = unpickler.load()
++ def getClassName(self, data):
++ deserializer = self._get_deserializer(data)
++ klass = deserializer.getClassMetadata(data)
+ if isinstance(klass, tuple):
+ klass, args = klass
+ if isinstance(klass, tuple):
+@@ -564,9 +554,10 @@
+ return "%s.%s" % klass
+ return "%s.%s" % (klass.__module__, klass.__name__)
+
+- def getGhost(self, pickle):
+- unpickler = self._get_unpickler(pickle)
+- klass = unpickler.load()
++ def getGhost(self, data):
++ deserializer = self._get_deserializer(data)
++ klass = deserializer.getClassMetadata(data)
++
+ if isinstance(klass, tuple):
+ # Here we have a separate class and args.
+ # This could be an old record, so the class module ne a named
+@@ -590,23 +581,22 @@
+
+ return klass.__new__(klass, *args)
+
+- def getState(self, pickle):
+- unpickler = self._get_unpickler(pickle)
++ def getState(self, data):
++ deserializer = self._get_deserializer(data)
+ try:
+- unpickler.load() # skip the class metadata
+- return unpickler.load()
++ return deserializer.getState(data)
+ except EOFError, msg:
+ log = logging.getLogger("ZODB.serialize")
+- log.exception("Unpickling error: %r", pickle)
++ log.exception("Deserialization error: %r", data)
+ raise
+
+- def setGhostState(self, obj, pickle):
+- state = self.getState(pickle)
++ def setGhostState(self, obj, p):
++ state = self.getState(p)
+ obj.__setstate__(state)
+
+
+ def referencesf(p, oids=None):
+- """Return a list of object ids found in a pickle
++ """Return a list of object ids found in a serialized object.
+
+ A list may be passed in, in which case, information is
+ appended to it.
+@@ -615,11 +605,8 @@
+ Weak and multi-database references are not included.
+ """
+
+- refs = []
+- u = cPickle.Unpickler(cStringIO.StringIO(p))
+- u.persistent_load = refs
+- u.noload()
+- u.noload()
++ sf = detect_format(p)
++ refs = sf.listPersistentReferences(p)
+
+ # Now we have a list of referencs. Need to convert to list of
+ # oids:
+@@ -644,20 +631,17 @@
+ 'w': lambda oid: None,
+ }
+
+-def get_refs(a_pickle):
+- """Return oid and class information for references in a pickle
++def get_refs(p):
++ """Return oid and class information for references in a serialized object.
+
+ The result of a list of oid and class information tuples.
+ If the reference doesn't contain class information, then the
+ klass information is None.
+ """
+-
+- refs = []
+- u = cPickle.Unpickler(cStringIO.StringIO(a_pickle))
+- u.persistent_load = refs
+- u.noload()
+- u.noload()
+
++ sf = detect_format(p)
++ refs = sf.listPersistentReferences(p)
++
+ # Now we have a list of referencs. Need to convert to list of
+ # oids and class info:
+
+Index: src/ZODB/format.py
+===================================================================
+--- src/ZODB/format.py (revision 0)
++++ src/ZODB/format.py (revision 0)
+@@ -0,0 +1,130 @@
++##############################################################################
++#
++# Copyright (c) 2009 Zope Corporation and Contributors.
++# All Rights Reserved.
++#
++# This software is subject to the provisions of the Zope Public License,
++# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
++# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
++# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
++# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
++# FOR A PARTICULAR PURPOSE.
++#
++##############################################################################
++"""Support for multiple object serialization formats in ZODB.
++
++This module implements the pickle serialization format, but it is
++possible to register other formats by calling register_format().
++"""
++
++import cPickle
++import cStringIO
++
++from ZODB.interfaces import ISerialFormat, ISerializer, IDeserializer
++from zope.interface import implements
++
++
++class PickleFormat(object):
++ """Implementation of ISerialFormat that reads/writes pickles"""
++ implements(ISerialFormat)
++
++ def makeSerializer(self, persistent_id):
++ return PickleSerializer(persistent_id)
++
++ def makeDeserializer(self, persistent_load, find_global=None):
++ return PickleDeserializer(persistent_load, find_global)
++
++ def listPersistentReferences(self, data):
++ refs = []
++ u = cPickle.Unpickler(cStringIO.StringIO(data))
++ u.persistent_load = refs
++ u.noload()
++ u.noload()
++ return refs
++
++
++class PickleSerializer(object):
++ """Implementation of ISerializer that dumps to pickles"""
++ implements(ISerializer)
++
++ def __init__(self, persistent_id):
++ self._file = cStringIO.StringIO()
++ self._p = cPickle.Pickler(self._file, 1)
++ self._p.persistent_id = persistent_id
++
++ def dump(self, classmeta, state):
++ # To reuse the existing cStringIO object, we must reset
++ # the file position to 0 and truncate the file after the
++ # new pickle is written.
++ self._file.seek(0)
++ self._p.clear_memo()
++ self._p.dump(classmeta)
++ self._p.dump(state)
++ self._file.truncate()
++ return self._file.getvalue()
++
++
++class PickleDeserializer(object):
++ """Implementation of IDeserializer that loads from a pair of pickles"""
++ implements(IDeserializer)
++
++ def __init__(self, persistent_load, find_global=None):
++ self.persistent_load = persistent_load
++ self.find_global = find_global
++
++ def getClassAndState(self, data):
++ unpickler = cPickle.Unpickler(cStringIO.StringIO(data))
++ unpickler.persistent_load = self.persistent_load
++ find_global = self.find_global
++ if find_global is not None:
++ unpickler.find_global = find_global
++ yield unpickler.load()
++ yield unpickler.load()
++
++ def getClassMetadata(self, data):
++ # load only the first pickle
++ return self.getClassAndState(data).next()
++
++ def getState(self, data):
++ i = self.getClassAndState(data)
++ i.next()
++ return i.next()
++
++# serial_formats: {name -> ISerialFormat provider}.
++serial_formats = {'': PickleFormat()}
++
++def register_format(name, impl):
++ """Register a serialization format.
++
++ name must be unique and impl must be an object that provides
++ ISerialFormat.
++ """
++ if name in serial_formats:
++ if serial_formats[name] is impl:
++ return
++ raise KeyError("Format %r is already registered" % name)
++ if not ISerialFormat.providedBy(impl):
++ raise ValueError("Object %r does not provide ISerialFormat" % impl)
++ serial_formats[name] = impl
++
++def detect_format(data):
++ """Detect the format of a serialized object and return an ISerialFormat.
++
++ Looks for a serial format name enclosed in curly braces at the
++ beginning of the data. Note that the opening curly brace character
++ is not currently a Python pickle opcode, so this should not conflict
++ with any Python pickle.
++
++ Returns an object that provides ISerialFormat.
++ """
++ if data.startswith('{'):
++ pos = data.find('}', 1)
++ if pos < 0:
++ raise ValueError('Serialized object has incomplete format name')
++ sf_name = data[1:pos]
++ return serial_formats[sf_name]
++ return serial_formats['']
++
++def get_format(name):
++ """Return the named serial format."""
++ return serial_formats[name]
+Index: src/ZODB/utils.py
+===================================================================
+--- src/ZODB/utils.py (revision 93829)
++++ src/ZODB/utils.py (working copy)
+@@ -17,14 +17,13 @@
+ import struct
+ from struct import pack, unpack
+ from binascii import hexlify, unhexlify
+-import cPickle as pickle
+-from cStringIO import StringIO
+ import weakref
+ import warnings
+ from tempfile import mkstemp
+ import os
+
+ from persistent.TimeStamp import TimeStamp
++from ZODB.format import detect_format
+
+ __all__ = ['z64',
+ 'p64',
+@@ -203,10 +202,14 @@
+ return modname, classname
+
+ # Else there are a bunch of other possible formats.
+- f = StringIO(data)
+- u = pickle.Unpickler(f)
++ def persistent_load(ref):
++ raise NotImplementedError(
++ "persistent_load not implemented in get_pickle_metadata")
++
++ sf = detect_format(data)
++ deserializer = sf.makeDeserializer(persistent_load)
+ try:
+- class_info = u.load()
++ class_info = deserializer.getClassMetadata(data)
+ except Exception, err:
+ print "Error", err
+ return '', ''
+Index: src/ZODB/ExportImport.py
+===================================================================
+--- src/ZODB/ExportImport.py (revision 93829)
++++ src/ZODB/ExportImport.py (working copy)
+@@ -15,8 +15,6 @@
+
+ import os
+
+-from cStringIO import StringIO
+-from cPickle import Pickler, Unpickler
+ from tempfile import TemporaryFile
+ import logging
+
+@@ -25,6 +23,7 @@
+ from ZODB.POSException import ExportError, POSKeyError
+ from ZODB.serialize import referencesf
+ from ZODB.utils import p64, u64, cp, mktemp
++from ZODB.format import detect_format
+
+ logger = logging.getLogger('ZODB.ExportImport')
+
+@@ -166,18 +165,12 @@
+ f.seek(-len(blob_begin_marker),1)
+ blob_filename = None
+
+- pfile = StringIO(data)
+- unpickler = Unpickler(pfile)
+- unpickler.persistent_load = persistent_load
++ sf = detect_format(data)
++ d = sf.makeDeserializer(persistent_load)
++ metadata, state = d.getClassAndState(data)
++ s = sf.makeSerializer(persistent_id)
++ data = s.dump(metadata, state)
+
+- newp = StringIO()
+- pickler = Pickler(newp, 1)
+- pickler.persistent_id = persistent_id
+-
+- pickler.dump(unpickler.load())
+- pickler.dump(unpickler.load())
+- data = newp.getvalue()
+-
+ if blob_filename is not None:
+ self._storage.storeBlob(oid, None, data, blob_filename,
+ version, transaction)
+Index: src/ZODB/ConflictResolution.py
+===================================================================
+--- src/ZODB/ConflictResolution.py (revision 93829)
++++ src/ZODB/ConflictResolution.py (working copy)
+@@ -13,14 +13,13 @@
+ ##############################################################################
+
+ import logging
+-from cStringIO import StringIO
+-from cPickle import Unpickler, Pickler
+ from pickle import PicklingError
+
+ import zope.interface
+
+ from ZODB.POSException import ConflictError
+ from ZODB.loglevels import BLATHER
++from ZODB.format import detect_format
+
+ logger = logging.getLogger('ZODB.ConflictResolution')
+
+@@ -53,12 +52,9 @@
+
+ def state(self, oid, serial, prfactory, p=''):
+ p = p or self.loadSerial(oid, serial)
+- file = StringIO(p)
+- unpickler = Unpickler(file)
+- unpickler.find_global = find_global
+- unpickler.persistent_load = prfactory.persistent_load
+- unpickler.load() # skip the class tuple
+- return unpickler.load()
++ sf = detect_format(p)
++ deserializer = sf.makeDeserializer(prfactory.persistent_load, find_global)
++ return deserializer.getState(p)
+
+ class IPersistentReference(zope.interface.Interface):
+ '''public contract for references to persistent objects from an object
+@@ -170,16 +166,16 @@
+ return object.data
+
+ _unresolvable = {}
+-def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
++def tryToResolveConflict(self, oid, committedSerial, oldSerial, newdata,
+ committedData=''):
+ # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
+ try:
+ prfactory = PersistentReferenceFactory()
+- file = StringIO(newpickle)
+- unpickler = Unpickler(file)
+- unpickler.find_global = find_global
+- unpickler.persistent_load = prfactory.persistent_load
+- meta = unpickler.load()
++ sf = detect_format(newdata)
++ deserializer = sf.makeDeserializer(
++ prfactory.persistent_load, find_global)
++ newdata_iter = deserializer.getClassAndState(newdata)
++ meta = newdata_iter.next()
+ if isinstance(meta, tuple):
+ klass = meta[0]
+ newargs = meta[1] or ()
+@@ -192,7 +188,7 @@
+ if klass in _unresolvable:
+ return None
+
+- newstate = unpickler.load()
++ newstate = newdata_iter.next()
+ inst = klass.__new__(klass, *newargs)
+
+ try:
+@@ -206,12 +202,8 @@
+
+ resolved = resolve(old, committed, newstate)
+
+- file = StringIO()
+- pickler = Pickler(file,1)
+- pickler.persistent_id = persistent_id
+- pickler.dump(meta)
+- pickler.dump(resolved)
+- return file.getvalue(1)
++ serializer = sf.makeSerializer(persistent_id)
++ return serializer.dump(meta, resolved)
+ except (ConflictError, BadClassName):
+ return None
+ except:
Property changes on: Sandbox/shane/serialization-format-ZODB-3-8.patch
___________________________________________________________________
Added: svn:mergeinfo
+
Added: Sandbox/shane/serialization-format-ZODB-trunk.patch
===================================================================
--- Sandbox/shane/serialization-format-ZODB-trunk.patch (rev 0)
+++ Sandbox/shane/serialization-format-ZODB-trunk.patch 2009-01-15 07:56:31 UTC (rev 94739)
@@ -0,0 +1,665 @@
+Index: src/ZODB/interfaces.py
+===================================================================
+--- src/ZODB/interfaces.py (revision 94738)
++++ src/ZODB/interfaces.py (working copy)
+@@ -1088,3 +1088,88 @@
+ """A combination of StopIteration and IndexError to provide a
+ backwards-compatible exception.
+ """
++
++
++class ISerialFormat(Interface):
++ """A method of serializing objects."""
++
++ def makeSerializer(persistent_id):
++ """Returns a new ISerializer provider.
++
++ The ISerializer must use the given persistent_id function.
++ """
++
++ def makeDeserializer(persistent_load, find_global=None):
++ """Returns a new IDeserializer provider.
++
++ The IDeserializer must use the given persistent_load and find_global
++ functions. The find_global parameter is optional.
++ """
++
++ def listPersistentReferences(data):
++ """Return the list of persistent references from a serialized object.
++
++ If a format name header was provided when the object was dumped,
++ that header will still be in the data parameter.
++
++ Each reference in the returned list may take any of the forms
++ described in the "Persistent References" section of the ZODB.serialize
++ module documentation, such as a simple oid string,
++ (oid, class meta data), or [reference_type, args].
++ """
++
++class ISerializer(Interface):
++ """Serializes objects."""
++
++ def dump(classmeta, state):
++ """Return serialized data given class metadata and an instance state.
++
++ The implementation of this method must use the persistent_id
++ function (provided by makeDumper()) when storing persistent
++ references. The persistent_id method both provides an identity
++ for targets and provides a way for the caller to generate a list of
++ persistent references.
++
++ The implementation needs to prepend a serialization format name
++ (in curly braces) unless this format is the default format.
++ """
++
++class IDeserializer(Interface):
++ """Deserializes objects."""
++
++ def getClassAndState(data):
++ """Return an iterator that produces the class metadata, then the state.
++
++ The recommended implementation is to use two yield statements,
++ making the method a generator.
++ """
++
++ def getClassMetadata(data):
++ """Return the class metadata portion of a serialized object.
++
++ If a format name header was provided when the object was dumped,
++ that header will still be in the data parameter.
++
++ The returned value can be any type of class metadata,
++ such as a class object or a tuple of (class, class_args);
++ the first element of that tuple can be a
++ (module_name, class_name) tuple instead of a class.
++
++ Implementations must use the persistent_load and find_global
++ functions provided to makeDeserializer() when they need to
++ resolve persistent references.
++ """
++
++ def getState(data):
++ """Return the state portion of a serialized object.
++
++ If a format name header was provided when the object was dumped,
++ that header will still be in the data parameter.
++
++ At some point, the returned state will be passed, unchanged,
++ to the __setstate__ method of an existing empty object.
++
++ Implementations must use the persistent_load and find_global
++ functions provided to makeDeserializer() when they need to
++ resolve persistent references.
++ """
+Index: src/ZODB/serialize.py
+===================================================================
+--- src/ZODB/serialize.py (revision 94738)
++++ src/ZODB/serialize.py (working copy)
+@@ -13,8 +13,9 @@
+ ##############################################################################
+ """Support for ZODB object serialization.
+
+-ZODB serializes objects using a custom format based on Python pickles.
+-When an object is unserialized, it can be loaded as either a ghost or
++Multiple types of object serialization are possible. The default
++serialization uses a custom format based on Python pickles.
++When an object is deserialized, it can be loaded as either a ghost or
+ a real object. A ghost is a persistent object of the appropriate type
+ but without any state. The first time a ghost is accessed, the
+ persistence machinery traps access and loads the actual state. A
+@@ -24,8 +25,8 @@
+ Pickle format
+ -------------
+
+-ZODB stores serialized objects using a custom format based on pickle.
+-Each serialized object has two parts: the class description and the
++By default, ZODB stores serialized objects using a custom format based on
++pickle. Each serialized object has two parts: the class description and the
+ object state. The class description must provide enough information
+ to call the class's ``__new__`` and create an empty object. Once the
+ object exists as a ghost, its state is passed to ``__setstate__``.
+@@ -133,16 +134,14 @@
+
+ """
+
+-import cPickle
+-import cStringIO
+ import logging
+
+-
+ from persistent import Persistent
+ from persistent.wref import WeakRefMarker, WeakRef
+ from ZODB import broken
+ from ZODB.broken import Broken
+ from ZODB.POSException import InvalidObjectReference
++from ZODB.format import detect_format, get_format
+
+ _oidtypes = str, type(None)
+
+@@ -163,17 +162,15 @@
+ class ObjectWriter:
+ """Serializes objects for storage in the database.
+
+- The ObjectWriter creates object pickles in the ZODB format. It
+- also detects new persistent objects reachable from the current
+- object.
++ Objects can specify what serialization format to use, but the
++ default format is the ZODB pickle format. This class also detects
++ new persistent objects reachable from the current object.
+ """
+
+ _jar = None
+
+ def __init__(self, obj=None):
+- self._file = cStringIO.StringIO()
+- self._p = cPickle.Pickler(self._file, 1)
+- self._p.inst_persistent_id = self.persistent_id
++ self._dumpers = {} # {serial format name -> dump method}
+ self._stack = []
+ if obj is not None:
+ self._stack.append(obj)
+@@ -268,16 +265,16 @@
+
+ # Most objects are not persistent. The following cheap test
+ # identifies most of them. For these, we return None,
+- # signalling that the object should be pickled normally.
++ # signalling that the object should be serialized normally.
+ if not isinstance(obj, (Persistent, type, WeakRef)):
+- # Not persistent, pickle normally
++ # Not persistent, serialize normally
+ return None
+
+ # Any persistent object must have an oid:
+ try:
+ oid = obj._p_oid
+ except AttributeError:
+- # Not persistent, pickle normally
++ # Not persistent, serialize normally
+ return None
+
+ if not (oid is None or isinstance(oid, str)):
+@@ -287,7 +284,7 @@
+ if hasattr(oid, '__get__'):
+ # The oid is a descriptor. That means obj is a non-persistent
+ # class whose instances are persistent, so ...
+- # Not persistent, pickle normally
++ # Not persistent, serialize normally
+ return None
+
+ if oid is WeakRefMarker:
+@@ -391,39 +388,36 @@
+ if (isinstance(getattr(klass, '_p_oid', 0), _oidtypes)
+ and klass.__module__):
+ # This is a persistent class with a non-empty module. This
+- # uses pickle format #3 or #7.
++ # uses class metadata format #3 or #7.
+ klass = klass.__module__, klass.__name__
+ if newargs is None:
+ meta = klass, None
+ else:
+ meta = klass, newargs()
+ elif newargs is None:
+- # Pickle format #1.
++ # Class metadata format #1.
+ meta = klass
+ else:
+- # Pickle format #2.
++ # Class metadata format #2.
+ meta = klass, newargs()
+
+- return self._dump(meta, obj.__getstate__())
++ state = obj.__getstate__()
+
+- def _dump(self, classmeta, state):
+- # To reuse the existing cStringIO object, we must reset
+- # the file position to 0 and truncate the file after the
+- # new pickle is written.
+- self._file.seek(0)
+- self._p.clear_memo()
+- self._p.dump(classmeta)
+- self._p.dump(state)
+- self._file.truncate()
+- return self._file.getvalue()
++ serial_format_name = getattr(state, 'serial_format', '')
++ dump = self._dumpers.get(serial_format_name)
++ if dump is None:
++ sf = get_format(serial_format_name)
++ dump = sf.makeSerializer(self.persistent_id).dump
++ self._dumpers[serial_format_name] = dump
++ return dump(meta, state)
+
+ def __iter__(self):
+ return NewObjectIterator(self._stack)
+
+ class NewObjectIterator:
+
+- # The pickler is used as a forward iterator when the connection
+- # is looking for new objects to pickle.
++ # The ObjectWriter is used as a forward iterator when the connection
++ # is looking for new objects to serialize.
+
+ def __init__(self, stack):
+ self._stack = stack
+@@ -448,20 +442,16 @@
+ def _get_class(self, module, name):
+ return self._factory(self._conn, module, name)
+
+- def _get_unpickler(self, pickle):
+- file = cStringIO.StringIO(pickle)
+- unpickler = cPickle.Unpickler(file)
+- unpickler.persistent_load = self._persistent_load
++ def _get_deserializer(self, p):
+ factory = self._factory
+ conn = self._conn
+
+ def find_global(modulename, name):
+ return factory(conn, modulename, name)
+
+- unpickler.find_global = find_global
++ sf = detect_format(p)
++ return sf.makeDeserializer(self._persistent_load, find_global)
+
+- return unpickler
+-
+ loaders = {}
+
+ def _persistent_load(self, reference):
+@@ -554,9 +544,9 @@
+
+ return obj
+
+- def getClassName(self, pickle):
+- unpickler = self._get_unpickler(pickle)
+- klass = unpickler.load()
++ def getClassName(self, data):
++ deserializer = self._get_deserializer(data)
++ klass = deserializer.getClassMetadata(data)
+ if isinstance(klass, tuple):
+ klass, args = klass
+ if isinstance(klass, tuple):
+@@ -564,9 +554,10 @@
+ return "%s.%s" % klass
+ return "%s.%s" % (klass.__module__, klass.__name__)
+
+- def getGhost(self, pickle):
+- unpickler = self._get_unpickler(pickle)
+- klass = unpickler.load()
++ def getGhost(self, data):
++ deserializer = self._get_deserializer(data)
++ klass = deserializer.getClassMetadata(data)
++
+ if isinstance(klass, tuple):
+ # Here we have a separate class and args.
+ # This could be an old record, so the class module ne a named
+@@ -590,23 +581,22 @@
+
+ return klass.__new__(klass, *args)
+
+- def getState(self, pickle):
+- unpickler = self._get_unpickler(pickle)
++ def getState(self, data):
++ deserializer = self._get_deserializer(data)
+ try:
+- unpickler.load() # skip the class metadata
+- return unpickler.load()
++ return deserializer.getState(data)
+ except EOFError, msg:
+ log = logging.getLogger("ZODB.serialize")
+- log.exception("Unpickling error: %r", pickle)
++ log.exception("Deserialization error: %r", data)
+ raise
+
+- def setGhostState(self, obj, pickle):
+- state = self.getState(pickle)
++ def setGhostState(self, obj, p):
++ state = self.getState(p)
+ obj.__setstate__(state)
+
+
+ def referencesf(p, oids=None):
+- """Return a list of object ids found in a pickle
++ """Return a list of object ids found in a serialized object.
+
+ A list may be passed in, in which case, information is
+ appended to it.
+@@ -615,11 +605,8 @@
+ Weak and multi-database references are not included.
+ """
+
+- refs = []
+- u = cPickle.Unpickler(cStringIO.StringIO(p))
+- u.persistent_load = refs
+- u.noload()
+- u.noload()
++ sf = detect_format(p)
++ refs = sf.listPersistentReferences(p)
+
+ # Now we have a list of referencs. Need to convert to list of
+ # oids:
+@@ -644,20 +631,17 @@
+ 'w': lambda oid: None,
+ }
+
+-def get_refs(a_pickle):
+- """Return oid and class information for references in a pickle
++def get_refs(p):
++ """Return oid and class information for references in a serialized object.
+
+ The result of a list of oid and class information tuples.
+ If the reference doesn't contain class information, then the
+ klass information is None.
+ """
+-
+- refs = []
+- u = cPickle.Unpickler(cStringIO.StringIO(a_pickle))
+- u.persistent_load = refs
+- u.noload()
+- u.noload()
+
++ sf = detect_format(p)
++ refs = sf.listPersistentReferences(p)
++
+ # Now we have a list of referencs. Need to convert to list of
+ # oids and class info:
+
+Index: src/ZODB/format.py
+===================================================================
+--- src/ZODB/format.py (revision 0)
++++ src/ZODB/format.py (revision 0)
+@@ -0,0 +1,130 @@
++##############################################################################
++#
++# Copyright (c) 2009 Zope Corporation and Contributors.
++# All Rights Reserved.
++#
++# This software is subject to the provisions of the Zope Public License,
++# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
++# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
++# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
++# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
++# FOR A PARTICULAR PURPOSE.
++#
++##############################################################################
++"""Support for multiple object serialization formats in ZODB.
++
++This module implements the pickle serialization format, but it is
++possible to register other formats by calling register_format().
++"""
++
++import cPickle
++import cStringIO
++
++from ZODB.interfaces import ISerialFormat, ISerializer, IDeserializer
++from zope.interface import implements
++
++
++class PickleFormat(object):
++ """Implementation of ISerialFormat that reads/writes pickles"""
++ implements(ISerialFormat)
++
++ def makeSerializer(self, persistent_id):
++ return PickleSerializer(persistent_id)
++
++ def makeDeserializer(self, persistent_load, find_global=None):
++ return PickleDeserializer(persistent_load, find_global)
++
++ def listPersistentReferences(self, data):
++ refs = []
++ u = cPickle.Unpickler(cStringIO.StringIO(data))
++ u.persistent_load = refs
++ u.noload()
++ u.noload()
++ return refs
++
++
++class PickleSerializer(object):
++ """Implementation of ISerializer that dumps to pickles"""
++ implements(ISerializer)
++
++ def __init__(self, persistent_id):
++ self._file = cStringIO.StringIO()
++ self._p = cPickle.Pickler(self._file, 1)
++ self._p.inst_persistent_id = persistent_id
++
++ def dump(self, classmeta, state):
++ # To reuse the existing cStringIO object, we must reset
++ # the file position to 0 and truncate the file after the
++ # new pickle is written.
++ self._file.seek(0)
++ self._p.clear_memo()
++ self._p.dump(classmeta)
++ self._p.dump(state)
++ self._file.truncate()
++ return self._file.getvalue()
++
++
++class PickleDeserializer(object):
++ """Implementation of IDeserializer that loads from a pair of pickles"""
++ implements(IDeserializer)
++
++ def __init__(self, persistent_load, find_global=None):
++ self.persistent_load = persistent_load
++ self.find_global = find_global
++
++ def getClassAndState(self, data):
++ unpickler = cPickle.Unpickler(cStringIO.StringIO(data))
++ unpickler.persistent_load = self.persistent_load
++ find_global = self.find_global
++ if find_global is not None:
++ unpickler.find_global = find_global
++ yield unpickler.load()
++ yield unpickler.load()
++
++ def getClassMetadata(self, data):
++ # load only the first pickle
++ return self.getClassAndState(data).next()
++
++ def getState(self, data):
++ i = self.getClassAndState(data)
++ i.next()
++ return i.next()
++
++# serial_formats: {name -> ISerialFormat provider}.
++serial_formats = {'': PickleFormat()}
++
++def register_format(name, impl):
++ """Register a serialization format.
++
++ name must be unique and impl must be an object that provides
++ ISerialFormat.
++ """
++ if name in serial_formats:
++ if serial_formats[name] is impl:
++ return
++ raise KeyError("Format %r is already registered" % name)
++ if not ISerialFormat.providedBy(impl):
++ raise ValueError("Object %r does not provide ISerialFormat" % impl)
++ serial_formats[name] = impl
++
++def detect_format(data):
++ """Detect the format of a serialized object and return an ISerialFormat.
++
++ Looks for a serial format name enclosed in curly braces at the
++ beginning of the data. Note that the opening curly brace character
++ is not currently a Python pickle opcode, so this should not conflict
++ with any Python pickle.
++
++ Returns an object that provides ISerialFormat.
++ """
++ if data.startswith('{'):
++ pos = data.find('}', 1)
++ if pos < 0:
++ raise ValueError('Serialized object has incomplete format name')
++ sf_name = data[1:pos]
++ return serial_formats[sf_name]
++ return serial_formats['']
++
++def get_format(name):
++ """Return the named serial format."""
++ return serial_formats[name]
+Index: src/ZODB/utils.py
+===================================================================
+--- src/ZODB/utils.py (revision 94738)
++++ src/ZODB/utils.py (working copy)
+@@ -17,14 +17,13 @@
+ import struct
+ from struct import pack, unpack
+ from binascii import hexlify, unhexlify
+-import cPickle as pickle
+-from cStringIO import StringIO
+ import weakref
+ import warnings
+ from tempfile import mkstemp
+ import os
+
+ from persistent.TimeStamp import TimeStamp
++from ZODB.format import detect_format
+
+ __all__ = ['z64',
+ 'p64',
+@@ -201,10 +200,14 @@
+ return modname, classname
+
+ # Else there are a bunch of other possible formats.
+- f = StringIO(data)
+- u = pickle.Unpickler(f)
++ def persistent_load(ref):
++ raise NotImplementedError(
++ "persistent_load not implemented in get_pickle_metadata")
++
++ sf = detect_format(data)
++ deserializer = sf.makeDeserializer(persistent_load)
+ try:
+- class_info = u.load()
++ class_info = deserializer.getClassMetadata(data)
+ except Exception, err:
+ print "Error", err
+ return '', ''
+Index: src/ZODB/ExportImport.py
+===================================================================
+--- src/ZODB/ExportImport.py (revision 94738)
++++ src/ZODB/ExportImport.py (working copy)
+@@ -15,8 +15,6 @@
+
+ import os
+
+-from cStringIO import StringIO
+-from cPickle import Pickler, Unpickler
+ from tempfile import TemporaryFile
+ import logging
+
+@@ -25,6 +23,7 @@
+ from ZODB.POSException import ExportError, POSKeyError
+ from ZODB.serialize import referencesf
+ from ZODB.utils import p64, u64, cp, mktemp
++from ZODB.format import detect_format
+
+ logger = logging.getLogger('ZODB.ExportImport')
+
+@@ -164,18 +163,12 @@
+ f.seek(-len(blob_begin_marker),1)
+ blob_filename = None
+
+- pfile = StringIO(data)
+- unpickler = Unpickler(pfile)
+- unpickler.persistent_load = persistent_load
++ sf = detect_format(data)
++ d = sf.makeDeserializer(persistent_load)
++ metadata, state = d.getClassAndState(data)
++ s = sf.makeSerializer(persistent_id)
++ data = s.dump(metadata, state)
+
+- newp = StringIO()
+- pickler = Pickler(newp, 1)
+- pickler.inst_persistent_id = persistent_id
+-
+- pickler.dump(unpickler.load())
+- pickler.dump(unpickler.load())
+- data = newp.getvalue()
+-
+ if blob_filename is not None:
+ self._storage.storeBlob(oid, None, data, blob_filename,
+ '', transaction)
+Index: src/ZODB/ConflictResolution.py
+===================================================================
+--- src/ZODB/ConflictResolution.py (revision 94738)
++++ src/ZODB/ConflictResolution.py (working copy)
+@@ -13,14 +13,13 @@
+ ##############################################################################
+
+ import logging
+-from cStringIO import StringIO
+-from cPickle import Unpickler, Pickler
+ from pickle import PicklingError
+
+ import zope.interface
+
+ from ZODB.POSException import ConflictError
+ from ZODB.loglevels import BLATHER
++from ZODB.format import detect_format
+
+ logger = logging.getLogger('ZODB.ConflictResolution')
+
+@@ -53,12 +52,9 @@
+
+ def state(self, oid, serial, prfactory, p=''):
+ p = p or self.loadSerial(oid, serial)
+- file = StringIO(p)
+- unpickler = Unpickler(file)
+- unpickler.find_global = find_global
+- unpickler.persistent_load = prfactory.persistent_load
+- unpickler.load() # skip the class tuple
+- return unpickler.load()
++ sf = detect_format(p)
++ deserializer = sf.makeDeserializer(prfactory.persistent_load, find_global)
++ return deserializer.getState(p)
+
+ class IPersistentReference(zope.interface.Interface):
+ '''public contract for references to persistent objects from an object
+@@ -170,16 +166,16 @@
+ return object.data
+
+ _unresolvable = {}
+-def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
++def tryToResolveConflict(self, oid, committedSerial, oldSerial, newdata,
+ committedData=''):
+ # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
+ try:
+ prfactory = PersistentReferenceFactory()
+- file = StringIO(newpickle)
+- unpickler = Unpickler(file)
+- unpickler.find_global = find_global
+- unpickler.persistent_load = prfactory.persistent_load
+- meta = unpickler.load()
++ sf = detect_format(newdata)
++ deserializer = sf.makeDeserializer(
++ prfactory.persistent_load, find_global)
++ newdata_iter = deserializer.getClassAndState(newdata)
++ meta = newdata_iter.next()
+ if isinstance(meta, tuple):
+ klass = meta[0]
+ newargs = meta[1] or ()
+@@ -192,7 +188,7 @@
+ if klass in _unresolvable:
+ return None
+
+- newstate = unpickler.load()
++ newstate = newdata_iter.next()
+ inst = klass.__new__(klass, *newargs)
+
+ try:
+@@ -206,12 +202,8 @@
+
+ resolved = resolve(old, committed, newstate)
+
+- file = StringIO()
+- pickler = Pickler(file,1)
+- pickler.inst_persistent_id = persistent_id
+- pickler.dump(meta)
+- pickler.dump(resolved)
+- return file.getvalue(1)
++ serializer = sf.makeSerializer(persistent_id)
++ return serializer.dump(meta, resolved)
+ except (ConflictError, BadClassName):
+ return None
+ except:
Deleted: Sandbox/shane/zodb-serialization-format.patch
===================================================================
--- Sandbox/shane/zodb-serialization-format.patch 2009-01-15 07:21:11 UTC (rev 94738)
+++ Sandbox/shane/zodb-serialization-format.patch 2009-01-15 07:56:31 UTC (rev 94739)
@@ -1,664 +0,0 @@
-Index: src/ZODB/interfaces.py
-===================================================================
---- src/ZODB/interfaces.py (revision 93829)
-+++ src/ZODB/interfaces.py (working copy)
-@@ -955,3 +1005,87 @@
-
- class BlobError(Exception):
- pass
-+
-+class ISerialFormat(Interface):
-+ """A method of serializing objects."""
-+
-+ def makeSerializer(persistent_id):
-+ """Returns a new ISerializer provider.
-+
-+ The ISerializer must use the given persistent_id function.
-+ """
-+
-+ def makeDeserializer(persistent_load, find_global=None):
-+ """Returns a new IDeserializer provider.
-+
-+ The IDeserializer must use the given persistent_load and find_global
-+ functions. The find_global parameter is optional.
-+ """
-+
-+ def listPersistentReferences(data):
-+ """Return the list of persistent references from a serialized object.
-+
-+ If a format name header was provided when the object was dumped,
-+ that header will still be in the data parameter.
-+
-+ Each reference in the returned list may take any of the forms
-+ described in the "Persistent References" section of the ZODB.serialize
-+ module documentation, such as a simple oid string,
-+ (oid, class meta data), or [reference_type, args].
-+ """
-+
-+class ISerializer(Interface):
-+ """Serializes objects."""
-+
-+ def dump(classmeta, state):
-+ """Return serialized data given class metadata and an instance state.
-+
-+ The implementation of this method must use the persistent_id
-+ function (provided by makeDumper()) when storing persistent
-+ references. The persistent_id method both provides an identity
-+ for targets and provides a way for the caller to generate a list of
-+ persistent references.
-+
-+ The implementation needs to prepend a serialization format name
-+ (in curly braces) unless this format is the default format.
-+ """
-+
-+class IDeserializer(Interface):
-+ """Deserializes objects."""
-+
-+ def getClassAndState(data):
-+ """Return an iterator that produces the class metadata, then the state.
-+
-+ The recommended implementation is to use two yield statements,
-+ making the method a generator.
-+ """
-+
-+ def getClassMetadata(data):
-+ """Return the class metadata portion of a serialized object.
-+
-+ If a format name header was provided when the object was dumped,
-+ that header will still be in the data parameter.
-+
-+ The returned value can be any type of class metadata,
-+ such as a class object or a tuple of (class, class_args);
-+ the first element of that tuple can be a
-+ (module_name, class_name) tuple instead of a class.
-+
-+ Implementations must use the persistent_load and find_global
-+ functions provided to makeDeserializer() when they need to
-+ resolve persistent references.
-+ """
-+
-+ def getState(data):
-+ """Return the state portion of a serialized object.
-+
-+ If a format name header was provided when the object was dumped,
-+ that header will still be in the data parameter.
-+
-+ At some point, the returned state will be passed, unchanged,
-+ to the __setstate__ method of an existing empty object.
-+
-+ Implementations must use the persistent_load and find_global
-+ functions provided to makeDeserializer() when they need to
-+ resolve persistent references.
-+ """
-Index: src/ZODB/serialize.py
-===================================================================
---- src/ZODB/serialize.py (revision 93829)
-+++ src/ZODB/serialize.py (working copy)
-@@ -13,8 +13,9 @@
- ##############################################################################
- """Support for ZODB object serialization.
-
--ZODB serializes objects using a custom format based on Python pickles.
--When an object is unserialized, it can be loaded as either a ghost or
-+Multiple types of object serialization are possible. The default
-+serialization uses a custom format based on Python pickles.
-+When an object is deserialized, it can be loaded as either a ghost or
- a real object. A ghost is a persistent object of the appropriate type
- but without any state. The first time a ghost is accessed, the
- persistence machinery traps access and loads the actual state. A
-@@ -24,8 +25,8 @@
- Pickle format
- -------------
-
--ZODB stores serialized objects using a custom format based on pickle.
--Each serialized object has two parts: the class description and the
-+By default, ZODB stores serialized objects using a custom format based on
-+pickle. Each serialized object has two parts: the class description and the
- object state. The class description must provide enough information
- to call the class's ``__new__`` and create an empty object. Once the
- object exists as a ghost, its state is passed to ``__setstate__``.
-@@ -133,16 +134,14 @@
-
- """
-
--import cPickle
--import cStringIO
- import logging
-
--
- from persistent import Persistent
- from persistent.wref import WeakRefMarker, WeakRef
- from ZODB import broken
- from ZODB.broken import Broken
- from ZODB.POSException import InvalidObjectReference
-+from ZODB.format import detect_format, get_format
-
- _oidtypes = str, type(None)
-
-@@ -163,17 +162,15 @@
- class ObjectWriter:
- """Serializes objects for storage in the database.
-
-- The ObjectWriter creates object pickles in the ZODB format. It
-- also detects new persistent objects reachable from the current
-- object.
-+ Objects can specify what serialization format to use, but the
-+ default format is the ZODB pickle format. This class also detects
-+ new persistent objects reachable from the current object.
- """
-
- _jar = None
-
- def __init__(self, obj=None):
-- self._file = cStringIO.StringIO()
-- self._p = cPickle.Pickler(self._file, 1)
-- self._p.persistent_id = self.persistent_id
-+ self._dumpers = {} # {serial format name -> dump method}
- self._stack = []
- if obj is not None:
- self._stack.append(obj)
-@@ -268,16 +265,16 @@
-
- # Most objects are not persistent. The following cheap test
- # identifies most of them. For these, we return None,
-- # signalling that the object should be pickled normally.
-+ # signalling that the object should be serialized normally.
- if not isinstance(obj, (Persistent, type, WeakRef)):
-- # Not persistent, pickle normally
-+ # Not persistent, serialize normally
- return None
-
- # Any persistent object must have an oid:
- try:
- oid = obj._p_oid
- except AttributeError:
-- # Not persistent, pickle normally
-+ # Not persistent, serialize normally
- return None
-
- if not (oid is None or isinstance(oid, str)):
-@@ -287,7 +284,7 @@
- if hasattr(oid, '__get__'):
- # The oid is a descriptor. That means obj is a non-persistent
- # class whose instances are persistent, so ...
-- # Not persistent, pickle normally
-+ # Not persistent, serialize normally
- return None
-
- if oid is WeakRefMarker:
-@@ -391,39 +388,36 @@
- if (isinstance(getattr(klass, '_p_oid', 0), _oidtypes)
- and klass.__module__):
- # This is a persistent class with a non-empty module. This
-- # uses pickle format #3 or #7.
-+ # uses class metadata format #3 or #7.
- klass = klass.__module__, klass.__name__
- if newargs is None:
- meta = klass, None
- else:
- meta = klass, newargs()
- elif newargs is None:
-- # Pickle format #1.
-+ # Class metadata format #1.
- meta = klass
- else:
-- # Pickle format #2.
-+ # Class metadata format #2.
- meta = klass, newargs()
-
-- return self._dump(meta, obj.__getstate__())
-+ state = obj.__getstate__()
-
-- def _dump(self, classmeta, state):
-- # To reuse the existing cStringIO object, we must reset
-- # the file position to 0 and truncate the file after the
-- # new pickle is written.
-- self._file.seek(0)
-- self._p.clear_memo()
-- self._p.dump(classmeta)
-- self._p.dump(state)
-- self._file.truncate()
-- return self._file.getvalue()
-+ serial_format_name = getattr(state, 'serial_format', '')
-+ dump = self._dumpers.get(serial_format_name)
-+ if dump is None:
-+ sf = get_format(serial_format_name)
-+ dump = sf.makeSerializer(self.persistent_id).dump
-+ self._dumpers[serial_format_name] = dump
-+ return dump(meta, state)
-
- def __iter__(self):
- return NewObjectIterator(self._stack)
-
- class NewObjectIterator:
-
-- # The pickler is used as a forward iterator when the connection
-- # is looking for new objects to pickle.
-+ # The ObjectWriter is used as a forward iterator when the connection
-+ # is looking for new objects to serialize.
-
- def __init__(self, stack):
- self._stack = stack
-@@ -448,20 +442,16 @@
- def _get_class(self, module, name):
- return self._factory(self._conn, module, name)
-
-- def _get_unpickler(self, pickle):
-- file = cStringIO.StringIO(pickle)
-- unpickler = cPickle.Unpickler(file)
-- unpickler.persistent_load = self._persistent_load
-+ def _get_deserializer(self, p):
- factory = self._factory
- conn = self._conn
-
- def find_global(modulename, name):
- return factory(conn, modulename, name)
-
-- unpickler.find_global = find_global
-+ sf = detect_format(p)
-+ return sf.makeDeserializer(self._persistent_load, find_global)
-
-- return unpickler
--
- loaders = {}
-
- def _persistent_load(self, reference):
-@@ -554,9 +544,9 @@
-
- return obj
-
-- def getClassName(self, pickle):
-- unpickler = self._get_unpickler(pickle)
-- klass = unpickler.load()
-+ def getClassName(self, data):
-+ deserializer = self._get_deserializer(data)
-+ klass = deserializer.getClassMetadata(data)
- if isinstance(klass, tuple):
- klass, args = klass
- if isinstance(klass, tuple):
-@@ -564,9 +554,10 @@
- return "%s.%s" % klass
- return "%s.%s" % (klass.__module__, klass.__name__)
-
-- def getGhost(self, pickle):
-- unpickler = self._get_unpickler(pickle)
-- klass = unpickler.load()
-+ def getGhost(self, data):
-+ deserializer = self._get_deserializer(data)
-+ klass = deserializer.getClassMetadata(data)
-+
- if isinstance(klass, tuple):
- # Here we have a separate class and args.
- # This could be an old record, so the class module ne a named
-@@ -590,23 +581,22 @@
-
- return klass.__new__(klass, *args)
-
-- def getState(self, pickle):
-- unpickler = self._get_unpickler(pickle)
-+ def getState(self, data):
-+ deserializer = self._get_deserializer(data)
- try:
-- unpickler.load() # skip the class metadata
-- return unpickler.load()
-+ return deserializer.getState(data)
- except EOFError, msg:
- log = logging.getLogger("ZODB.serialize")
-- log.exception("Unpickling error: %r", pickle)
-+ log.exception("Deserialization error: %r", data)
- raise
-
-- def setGhostState(self, obj, pickle):
-- state = self.getState(pickle)
-+ def setGhostState(self, obj, p):
-+ state = self.getState(p)
- obj.__setstate__(state)
-
-
- def referencesf(p, oids=None):
-- """Return a list of object ids found in a pickle
-+ """Return a list of object ids found in a serialized object.
-
- A list may be passed in, in which case, information is
- appended to it.
-@@ -615,11 +605,8 @@
- Weak and multi-database references are not included.
- """
-
-- refs = []
-- u = cPickle.Unpickler(cStringIO.StringIO(p))
-- u.persistent_load = refs
-- u.noload()
-- u.noload()
-+ sf = detect_format(p)
-+ refs = sf.listPersistentReferences(p)
-
- # Now we have a list of referencs. Need to convert to list of
- # oids:
-@@ -644,20 +631,17 @@
- 'w': lambda oid: None,
- }
-
--def get_refs(a_pickle):
-- """Return oid and class information for references in a pickle
-+def get_refs(p):
-+ """Return oid and class information for references in a serialized object.
-
- The result of a list of oid and class information tuples.
- If the reference doesn't contain class information, then the
- klass information is None.
- """
--
-- refs = []
-- u = cPickle.Unpickler(cStringIO.StringIO(a_pickle))
-- u.persistent_load = refs
-- u.noload()
-- u.noload()
-
-+ sf = detect_format(p)
-+ refs = sf.listPersistentReferences(p)
-+
- # Now we have a list of referencs. Need to convert to list of
- # oids and class info:
-
-Index: src/ZODB/format.py
-===================================================================
---- src/ZODB/format.py (revision 0)
-+++ src/ZODB/format.py (revision 0)
-@@ -0,0 +1,130 @@
-+##############################################################################
-+#
-+# Copyright (c) 2009 Zope Corporation and Contributors.
-+# All Rights Reserved.
-+#
-+# This software is subject to the provisions of the Zope Public License,
-+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-+# FOR A PARTICULAR PURPOSE.
-+#
-+##############################################################################
-+"""Support for multiple object serialization formats in ZODB.
-+
-+This module implements the pickle serialization format, but it is
-+possible to register other formats by calling register_format().
-+"""
-+
-+import cPickle
-+import cStringIO
-+
-+from ZODB.interfaces import ISerialFormat, ISerializer, IDeserializer
-+from zope.interface import implements
-+
-+
-+class PickleFormat(object):
-+ """Implementation of ISerialFormat that reads/writes pickles"""
-+ implements(ISerialFormat)
-+
-+ def makeSerializer(self, persistent_id):
-+ return PickleSerializer(persistent_id)
-+
-+ def makeDeserializer(self, persistent_load, find_global=None):
-+ return PickleDeserializer(persistent_load, find_global)
-+
-+ def listPersistentReferences(self, data):
-+ refs = []
-+ u = cPickle.Unpickler(cStringIO.StringIO(data))
-+ u.persistent_load = refs
-+ u.noload()
-+ u.noload()
-+ return refs
-+
-+
-+class PickleSerializer(object):
-+ """Implementation of ISerializer that dumps to pickles"""
-+ implements(ISerializer)
-+
-+ def __init__(self, persistent_id):
-+ self._file = cStringIO.StringIO()
-+ self._p = cPickle.Pickler(self._file, 1)
-+ self._p.persistent_id = persistent_id
-+
-+ def dump(self, classmeta, state):
-+ # To reuse the existing cStringIO object, we must reset
-+ # the file position to 0 and truncate the file after the
-+ # new pickle is written.
-+ self._file.seek(0)
-+ self._p.clear_memo()
-+ self._p.dump(classmeta)
-+ self._p.dump(state)
-+ self._file.truncate()
-+ return self._file.getvalue()
-+
-+
-+class PickleDeserializer(object):
-+ """Implementation of IDeserializer that loads from a pair of pickles"""
-+ implements(IDeserializer)
-+
-+ def __init__(self, persistent_load, find_global=None):
-+ self.persistent_load = persistent_load
-+ self.find_global = find_global
-+
-+ def getClassAndState(self, data):
-+ unpickler = cPickle.Unpickler(cStringIO.StringIO(data))
-+ unpickler.persistent_load = self.persistent_load
-+ find_global = self.find_global
-+ if find_global is not None:
-+ unpickler.find_global = find_global
-+ yield unpickler.load()
-+ yield unpickler.load()
-+
-+ def getClassMetadata(self, data):
-+ # load only the first pickle
-+ return self.getClassAndState(data).next()
-+
-+ def getState(self, data):
-+ i = self.getClassAndState(data)
-+ i.next()
-+ return i.next()
-+
-+# serial_formats: {name -> ISerialFormat provider}.
-+serial_formats = {'': PickleFormat()}
-+
-+def register_format(name, impl):
-+ """Register a serialization format.
-+
-+ name must be unique and impl must be an object that provides
-+ ISerialFormat.
-+ """
-+ if name in serial_formats:
-+ if serial_formats[name] is impl:
-+ return
-+ raise KeyError("Format %r is already registered" % name)
-+ if not ISerialFormat.providedBy(impl):
-+ raise ValueError("Object %r does not provide ISerialFormat" % impl)
-+ serial_formats[name] = impl
-+
-+def detect_format(data):
-+ """Detect the format of a serialized object and return an ISerialFormat.
-+
-+ Looks for a serial format name enclosed in curly braces at the
-+ beginning of the data. Note that the opening curly brace character
-+ is not currently a Python pickle opcode, so this should not conflict
-+ with any Python pickle.
-+
-+ Returns an object that provides ISerialFormat.
-+ """
-+ if data.startswith('{'):
-+ pos = data.find('}', 1)
-+ if pos < 0:
-+ raise ValueError('Serialized object has incomplete format name')
-+ sf_name = data[1:pos]
-+ return serial_formats[sf_name]
-+ return serial_formats['']
-+
-+def get_format(name):
-+ """Return the named serial format."""
-+ return serial_formats[name]
-Index: src/ZODB/utils.py
-===================================================================
---- src/ZODB/utils.py (revision 93829)
-+++ src/ZODB/utils.py (working copy)
-@@ -17,14 +17,13 @@
- import struct
- from struct import pack, unpack
- from binascii import hexlify, unhexlify
--import cPickle as pickle
--from cStringIO import StringIO
- import weakref
- import warnings
- from tempfile import mkstemp
- import os
-
- from persistent.TimeStamp import TimeStamp
-+from ZODB.format import detect_format
-
- __all__ = ['z64',
- 'p64',
-@@ -203,10 +202,14 @@
- return modname, classname
-
- # Else there are a bunch of other possible formats.
-- f = StringIO(data)
-- u = pickle.Unpickler(f)
-+ def persistent_load(ref):
-+ raise NotImplementedError(
-+ "persistent_load not implemented in get_pickle_metadata")
-+
-+ sf = detect_format(data)
-+ deserializer = sf.makeDeserializer(persistent_load)
- try:
-- class_info = u.load()
-+ class_info = deserializer.getClassMetadata(data)
- except Exception, err:
- print "Error", err
- return '', ''
-Index: src/ZODB/ExportImport.py
-===================================================================
---- src/ZODB/ExportImport.py (revision 93829)
-+++ src/ZODB/ExportImport.py (working copy)
-@@ -15,8 +15,6 @@
-
- import os
-
--from cStringIO import StringIO
--from cPickle import Pickler, Unpickler
- from tempfile import TemporaryFile
- import logging
-
-@@ -25,6 +23,7 @@
- from ZODB.POSException import ExportError, POSKeyError
- from ZODB.serialize import referencesf
- from ZODB.utils import p64, u64, cp, mktemp
-+from ZODB.format import detect_format
-
- logger = logging.getLogger('ZODB.ExportImport')
-
-@@ -166,18 +165,12 @@
- f.seek(-len(blob_begin_marker),1)
- blob_filename = None
-
-- pfile = StringIO(data)
-- unpickler = Unpickler(pfile)
-- unpickler.persistent_load = persistent_load
-+ sf = detect_format(data)
-+ d = sf.makeDeserializer(persistent_load)
-+ metadata, state = d.getClassAndState(data)
-+ s = sf.makeSerializer(persistent_id)
-+ data = s.dump(metadata, state)
-
-- newp = StringIO()
-- pickler = Pickler(newp, 1)
-- pickler.persistent_id = persistent_id
--
-- pickler.dump(unpickler.load())
-- pickler.dump(unpickler.load())
-- data = newp.getvalue()
--
- if blob_filename is not None:
- self._storage.storeBlob(oid, None, data, blob_filename,
- version, transaction)
-Index: src/ZODB/ConflictResolution.py
-===================================================================
---- src/ZODB/ConflictResolution.py (revision 93829)
-+++ src/ZODB/ConflictResolution.py (working copy)
-@@ -13,14 +13,13 @@
- ##############################################################################
-
- import logging
--from cStringIO import StringIO
--from cPickle import Unpickler, Pickler
- from pickle import PicklingError
-
- import zope.interface
-
- from ZODB.POSException import ConflictError
- from ZODB.loglevels import BLATHER
-+from ZODB.format import detect_format
-
- logger = logging.getLogger('ZODB.ConflictResolution')
-
-@@ -53,12 +52,9 @@
-
- def state(self, oid, serial, prfactory, p=''):
- p = p or self.loadSerial(oid, serial)
-- file = StringIO(p)
-- unpickler = Unpickler(file)
-- unpickler.find_global = find_global
-- unpickler.persistent_load = prfactory.persistent_load
-- unpickler.load() # skip the class tuple
-- return unpickler.load()
-+ sf = detect_format(p)
-+ deserializer = sf.makeDeserializer(prfactory.persistent_load, find_global)
-+ return deserializer.getState(p)
-
- class IPersistentReference(zope.interface.Interface):
- '''public contract for references to persistent objects from an object
-@@ -170,16 +166,16 @@
- return object.data
-
- _unresolvable = {}
--def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
-+def tryToResolveConflict(self, oid, committedSerial, oldSerial, newdata,
- committedData=''):
- # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
- try:
- prfactory = PersistentReferenceFactory()
-- file = StringIO(newpickle)
-- unpickler = Unpickler(file)
-- unpickler.find_global = find_global
-- unpickler.persistent_load = prfactory.persistent_load
-- meta = unpickler.load()
-+ sf = detect_format(newdata)
-+ deserializer = sf.makeDeserializer(
-+ prfactory.persistent_load, find_global)
-+ newdata_iter = deserializer.getClassAndState(newdata)
-+ meta = newdata_iter.next()
- if isinstance(meta, tuple):
- klass = meta[0]
- newargs = meta[1] or ()
-@@ -192,7 +188,7 @@
- if klass in _unresolvable:
- return None
-
-- newstate = unpickler.load()
-+ newstate = newdata_iter.next()
- inst = klass.__new__(klass, *newargs)
-
- try:
-@@ -206,12 +202,8 @@
-
- resolved = resolve(old, committed, newstate)
-
-- file = StringIO()
-- pickler = Pickler(file,1)
-- pickler.persistent_id = persistent_id
-- pickler.dump(meta)
-- pickler.dump(resolved)
-- return file.getvalue(1)
-+ serializer = sf.makeSerializer(persistent_id)
-+ return serializer.dump(meta, resolved)
- except (ConflictError, BadClassName):
- return None
- except:
More information about the Checkins
mailing list