[Zodb-checkins] CVS: ZODB4/ZODB - Serialize.py:1.1

[Jeremy Hylton]

Update of /cvs-repository/ZODB4/ZODB
In directory cvs.zope.org:/tmp/cvs-serv22270/ZODB

Added Files:
	Serialize.py 
Log Message:
Add new module that handles all serialization.

The current implementation actually only does the unpickling.




=== Added File ZODB4/ZODB/Serialize.py ===
"""Support for ZODB object serialization.

ZODB serializes objects using a custom format based on Python pickles.
When an object is unserialized, it can be loaded as either a ghost or
a real object.  A ghost is a persistent object of the appropriate type
but without any state.  The first time a ghost is accessed, the
persistence machinery traps access and loads the actual state.  A
ghost allows many persistent objects to be loaded while minimizing the
memory consumption of referenced but otherwise unused objects.

Pickle format
-------------

ZODB pickles objects using a custom format.  Each object pickle had
two parts: the class description and the object state.  The class
description must provide enough information to call the class's
``__new__`` and create an empty object.  Once the object exists, its
state is passed to ``__getstate__``.

The class metadata is a three-tuple contained the module name, the
class name, and a tuple of arguments to pass to ``__new__``.  The last
element may be None if the only argument to ``__new__`` is the class.
"""

from cStringIO import StringIO
import cPickle
from types import StringType, TupleType

def getClass(module, name):
    mod = __import__(module)
    parts = module.split(".")
    for part in parts[1:]:
        mod = getattr(mod, part)
    return getattr(mod, name)

def getClassMetadata(object):
    klass = object.__class__
    module = klass.__module__
    classname = klass.__name__
    if hasattr(object, "__getnewargs__"):
        newargs = object.__getnewargs__()
    else:
        newargs = None
    return module, classname, newargs

class Pickler:
    pass

class BaseUnpickler:

    # subclasses must define _persistent_load().

    def _get_unpickler(self, pickle):
        file = StringIO(pickle)
        unpickler = cPickle.Unpickler(file)
        unpickler.persistent_load = self._persistent_load
        return unpickler
        
    def _new_object(self, module, classname, newargs=None):
        klass = getClass(module, classname)
        if newargs is None:
            object = klass.__new__(klass)
        else:
            object = klass.__new__(klass, *newargs)
                
        return object

    def getGhost(self, pickle):
        unpickler = self._get_unpickler(pickle)
        module, classname, newargs = unpickler.load()
        return self._new_object(module, classname, newargs)

    def getState(self, pickle):
        unpickler = self._get_unpickler(pickle)
        unpickler.load() # skip the class metadata
        state = unpickler.load()
        return state

    def setGhostState(self, object, pickle):
        state = self.getState(pickle)
        object.__setstate__(state)

class ConnectionUnpickler(BaseUnpickler):

    def __init__(self, conn, cache):
        self._conn = conn
        self._cache = cache

    def _persistent_load(self, oid):
        # persistent_load function to pass to Unpickler
        if isinstance(oid, TupleType):
            # XXX We get here via new_persistent_id()
            
            # Quick instance reference.  We know all we need to know
            # to create the instance w/o hitting the db, so go for it!
            oid, classmeta = oid
            object = self._cache.get(oid)
            if object is not None:
                return object

            object = self._new_object(*classmeta)

            # XXX should be done by connection
            object._p_oid = oid
            object._p_jar = self._conn
            object._p_changed = None
            
            self._cache[oid] = object
            return object

        object = self._cache.get(oid)
        if object is not None:
            return object
        return self._conn[oid]

class ResolveUnpickler(BaseUnpickler):

    bad_classes = {}

    def __init__(self, persistent_load):
        self._persistent_load = persistent_load

    def getGhost(self, pickle):
        unpickler = self._get_unpickler(pickle)
        classmeta = unpickler.load()
        if classmeta in self.bad_classes:
            return None
        else:
            return self._new_object(*classmeta)

    def getResolveMethod(self, pickle):
        ghost = self.getGhost(pickle)
        if ghost is None:
            return None
        resolve = getattr(ghost, "_p_resolveConflict", None)
        if resolve is None:
            # XXX too bad.  we just had classmeta.
            classmeta = getClassMeta(ghost)
            self.bad_classes[classmeta] = True
            return None
        else:
            return resolve

def new_persistent_id(self, stack):
    # XXX need a doc string.  not sure if the one for persistent_id()
    # below is correct.

    # Create a special persistent_id that captures T and the subobject
    # stack in a closure.

    def persistent_id(object):
        """Test if an object is persistent, returning an oid if it is.

        This function is used by the pickler to test whether an object
        is persistent.  If it isn't, the function returns None and the
        object is included in the pickle for the current persistent
        object.

        If it is persistent, it returns the oid and sometimes a tuple
        with other stuff.
        """
        if not hasattr(object, '_p_oid'):
            return None
        
        oid = object._p_oid

        # I'd like to write something like this --
        # if isinstance(oid, types.MemberDescriptor):
        # -- but I can't because the type doesn't have a canonical name.
        # Instead, we'll assert that an oid must always be a string
        if not (isinstance(oid, StringType) or oid is None):
            return None

        if oid is None or object._p_jar is not self:
            oid = self.new_oid()
            object._p_jar = self
            object._p_oid = oid
            stack.append(object)

        return oid, getClassMetadata(object)
    
    return persistent_id