[Zodb-checkins] CVS: Zope3/lib/python/Persistence - Class.py:1.7 Module.py:1.32 cPersistence.c:1.27 patch.py:1.3
Jeremy Hylton
jeremy@zope.com
Fri, 22 Nov 2002 00:33:58 -0500
Update of /cvs-repository/Zope3/lib/python/Persistence
In directory cvs.zope.org:/tmp/cvs-serv11127/Persistence
Modified Files:
Class.py Module.py cPersistence.c patch.py
Log Message:
Get rid of the persistent class dead chicken.
This change adds yet another layer of complexity to the persistent
module support. The patch module uses a pickler to copy the entire
persistent module, updating classes and functions along the way.
To update persistent classes in place, we need to use a special
__newstate__() method instead of __setstate__(), because a class's
__getstate__() only returns the persistent object and update-in-place
needs to handle all attributes (including removal of attributes that
no longer exist).
The update-in-place machinery tries to do something sensible in the
presence of aliasing, but it is probably half baked. Need to think
more about whether sensible semantics are possible, and implement them
if they are.
To handle updates to methods, we need to add PersistentDescriptor and
PersistentMethod objects that make PersistentFunctions behave like
functions when found in a class dict.
=== Zope3/lib/python/Persistence/Class.py 1.6 => 1.7 ===
--- Zope3/lib/python/Persistence/Class.py:1.6 Thu Oct 10 17:36:18 2002
+++ Zope3/lib/python/Persistence/Class.py Fri Nov 22 00:33:58 2002
@@ -26,7 +26,9 @@
# separate sets of attributes. This code should be documented, as it
# it quite delicate, and it should be move to a separate module.
-class ExtClassDescr(object):
+__metaclass__ = type
+
+class ExtClassDescr:
"""Maintains seperate class and instance descriptors for an attribute.
This allows a class to provide methods and attributes without
@@ -70,7 +72,7 @@
def clsdelete(self):
pass
-class MethodMixin(object):
+class MethodMixin:
def __init__(self, name, descr, func):
super(MethodMixin, self).__init__(name, descr)
@@ -85,7 +87,7 @@
raise
return f
-class DataMixin(object):
+class DataMixin:
def __init__(self, name, descr, val):
super(DataMixin, self).__init__(name, descr)
@@ -100,7 +102,7 @@
def clsdelete(self):
del self.val
-class ExtClassObject(object):
+class ExtClassObject:
_missing = object()
@@ -136,6 +138,62 @@
class ExtClassDataDescr(DataMixin, ExtClassDescr):
pass
+# The next three classes conspire to make a PersistentFunction
+# behave like a method when found in a class's __dict__.
+
+class PersistentMethod:
+ """Make PersistentFunctions into methods."""
+ def __init__(self, klass, inst, func):
+ self.im_class = klass
+ self.im_self = inst
+ self.im_func = func
+
+ def __repr__(self):
+ if self.im_self is None:
+ kind = "unbound"
+ else:
+ kind = "bound"
+ return ("<persistent %s method %s.%s of %s>"
+ % (kind, self.im_class.__name__, self.im_func.__name__,
+ self.im_self))
+
+ def __call__(self, *args, **kwargs):
+ if self.im_self is None:
+ if not isinstance(args[0], self.im_class):
+ raise TypeError("unbound method %s() must be called "
+ "with %s instance as first argument ("
+ "got %s instead)" % (self.im_func.__name__,
+ self.im_class.__name__,
+ type(args[0]).__name__))
+ else:
+ return self.im_func(self.im_self, *args, **kwargs)
+
+class PersistentDescriptor:
+
+ def __init__(self, objclass, func):
+ self.__name__ = func.__name__
+ self.__doc__ = func.__doc__
+ self.__objclass__ = objclass
+ self._func = func
+ # Delegate __getstate__ and __setstate__ to the persistent func.
+ # The patch module will use these methods to update persistent
+ # methods in place.
+ self.__getstate__ = func.__getstate__
+ self.__setstate__ = func.__setstate__
+
+ def __repr__(self):
+ return "<descriptor %s.%s>" % (self.__objclass__.__name__,
+ self.__name__)
+
+ def __get__(self, object, klass=None):
+ if object is None:
+ return PersistentMethod(klass or self.__objclass__, None,
+ self._func)
+ else:
+ return PersistentMethod(klass or self.__objclass__, object,
+ self._func)
+
+
# XXX is missing necessary for findattr?
# None might be sufficient
_missing = object()
@@ -162,7 +220,8 @@
_pc_init = 0
def __new__(meta, name, bases, dict):
- cls = super(PersistentClassMetaClass, meta).__new__(meta, name, bases, dict)
+ cls = super(PersistentClassMetaClass, meta).__new__(
+ meta, name, bases, dict)
# helper functions
def extend_attr(attr, v):
prev = findattr(cls, attr, None)
@@ -179,6 +238,10 @@
extend_meth("__getstate__", meta.__getstate__)
extend_meth("__setstate__", meta.__setstate__)
extend_attr("__implements__", meta.__implements__)
+
+ for k, v in dict.items():
+ if isinstance(v, PersistentFunction):
+ setattr(cls, k, PersistentDescriptor(cls, v))
cls._pc_init = 1
return cls
@@ -227,6 +290,12 @@
for k, v in dict.items():
setattr(cls, k, v)
+ # XXX hack method to support update in place
+
+ def __newstate__(cls, acls):
+ cls.__dict__.clear()
+ cls.__dict__.update(acls.__dict__)
+
def _p_deactivate(cls):
# do nothing but mark the state change for now
cls._p_state = GHOST
@@ -243,38 +312,3 @@
def __getnewargs__(cls):
return cls.__name__, cls.__bases__, {}
-
-class PersistentBaseClass(Persistent):
-
- __metaclass__ = PersistentClassMetaClass
-
-def _test():
- global PC, P
-
- class PC(Persistent):
-
- __metaclass__ = PersistentClassMetaClass
-
- def __init__(self):
- self.x = 1
-
- def inc(self):
- self.x += 1
-
- def __int__(self):
- return self.x
-
- class P(PersistentBaseClass):
-
- def __init__(self):
- self.x = 1
-
- def inc(self):
- self.x += 1
-
- def __int__(self):
- return self.x
-
-if __name__ == "__main__":
- _test()
-
=== Zope3/lib/python/Persistence/Module.py 1.31 => 1.32 ===
--- Zope3/lib/python/Persistence/Module.py:1.31 Thu Oct 10 18:26:05 2002
+++ Zope3/lib/python/Persistence/Module.py Fri Nov 22 00:33:58 2002
@@ -27,6 +27,7 @@
from Persistence.IPersistentModuleManager import IPersistentModuleManager
from Persistence.IPersistentModuleRegistry \
import IPersistentModuleImportRegistry, IPersistentModuleUpdateRegistry
+from Persistence.patch import NameFinder, convert
from Transaction import get_transaction
@@ -133,11 +134,13 @@
def update(self, source):
self._module._p_changed = True
moddict = self._module.__dict__
- copy = moddict.copy()
+ old_names = NameFinder(self._module)
moddict[__persistent_module_registry__] = self._registry
exec source in moddict
del moddict[__persistent_module_registry__]
- self._fixup(moddict, copy, self._module)
+ new_names = NameFinder(self._module)
+ replacements = new_names.replacements(old_names)
+ convert(self._module, replacements)
self.source = source
def remove(self, source):
=== Zope3/lib/python/Persistence/cPersistence.c 1.26 => 1.27 ===
--- Zope3/lib/python/Persistence/cPersistence.c:1.26 Thu Nov 7 13:48:30 2002
+++ Zope3/lib/python/Persistence/cPersistence.c Fri Nov 22 00:33:58 2002
@@ -154,7 +154,8 @@
/* XXX What's the contract of __setstate__() if the object already has
state? Should it update the state or should it replace the state?
- For now, I've decided to update the state.
+ A class to __setstate__() seems most likely intended to replace the
+ old state with a new state, so clear first.
*/
static PyObject *
@@ -176,6 +177,8 @@
if ((*pdict) == NULL)
return NULL;
}
+ else
+ PyDict_Clear(*pdict);
dict = *pdict;
if (!PyDict_Check(state)) {
=== Zope3/lib/python/Persistence/patch.py 1.2 => 1.3 ===
--- Zope3/lib/python/Persistence/patch.py:1.2 Thu Nov 21 15:32:12 2002
+++ Zope3/lib/python/Persistence/patch.py Fri Nov 22 00:33:58 2002
@@ -37,6 +37,10 @@
Implementation notes:
+The conversion operation is implemented using a pickler. It wasn't
+possible to use the copy module, because it isn't possible to extend
+the copy module in a safe way. The copy module depends on module globals.
+
What semantics do we want for update-in-place in the presence of aliases?
Semantics based on per-namespace updates don't work in the presence of
@@ -52,50 +56,97 @@
alias. When the class is updated, the alias changes, but the bound
method isn't. Then the bound method can invoke an old method on a new
object, which may not be legal. It might sufficient to outlaw this case.
+
+XXX Open issues
+
+Can we handle metaclasses within this framework? That is, what if an
+object's type is not type, but a subclass of type.
+
+How do we handle things like staticmethods? We'd like the code to be
+able to use them, but Python doesn't expose an introspection on them.
+
+What if the same object is bound to two different names in the same
+namespace? Example:
+ x = lambda: 1
+ y = x
+If the module is updated to:
+ x = lambda: 1
+ y = lambda: 2
+What are the desired semantics?
"""
__metaclass__ = type
+from copy_reg import dispatch_table
from cStringIO import StringIO
import pickle
from types import *
-from Persistence.Class import PersistentClassMetaClass
+from Persistence.Class import PersistentClassMetaClass, PersistentDescriptor
from Persistence.Function import PersistentFunction
+from Persistence import Persistent
-class FunctionWrapper:
+class Wrapper:
+ """Implement pickling reduce protocol for update-able object.
- def __init__(self, func, module):
- self._func = func
- self._module = module
- self._p_oid = id(func)
+ The Pickler creates a Wrapper instance and uses it as the reduce
+ function. The Unpickler calls the instance to recreate the
+ object.
+ """
+ __safe_for_unpickling__ = True
- def __call__(self, defaults, dict):
- self._func.func_defaults = defaults
- self._func.func_dict.update(dict)
- return PersistentFunction(self._func, self._module)
+ def __init__(self, obj, module, replace=None):
+ self._obj = obj
+ self._module = module
+ self._replace = replace
-class TypeWrapper:
+ def __call__(self, *args):
+ new = self.unwrap(*args)
+ if self._replace is not None:
+ # XXX Hack: Use __newstate__ for persistent classes, because
+ # a persistent class's persistent state is a fairly limited
+ # subset of the dict and we really want to replace everything.
+ if hasattr(self._replace, "__newstate__"):
+ self._replace.__newstate__(new)
+ else:
+ self._replace.__setstate__(new.__getstate__())
+ return self._replace
+ else:
+ return new
- def __init__(self, atype, module):
- self._type = atype
- self._module = module
+class FunctionWrapper(Wrapper):
- def __call__(self, bases, dict):
- return PersistentClassMetaClass(self._type.__name__, bases, dict)
+ def unwrap(self, defaults, dict):
+ self._obj.func_defaults = defaults
+ self._obj.func_dict.update(dict)
+ return PersistentFunction(self._obj, self._module)
+
+class TypeWrapper(Wrapper):
+
+ def unwrap(self, bases, dict):
+ # Must add Persistent to the list of bases so that type (the
+ # base class of PersistentClassMetaClass) will create the
+ # correct C layout.
+ return PersistentClassMetaClass(self._obj.__name__,
+ bases + (Persistent,), dict)
class Pickler(pickle.Pickler):
dispatch = {}
dispatch.update(pickle.Pickler.dispatch)
- def __init__(self, file, module, memo):
+ def __init__(self, file, module, memo, replacements):
pickle.Pickler.__init__(self, file, bin=True)
self._pmemo = memo
self._module = module
+ self._repl = replacements
+ self._builtins = module.__builtins__
+
+ def wrap(self, wrapperclass, object):
+ return wrapperclass(object, self._module, self._repl.get(id(object)))
- def persistent_id(self, object):
- if hasattr(object, "_p_oid"):
+ def persistent_id(self, object, force=False):
+ if isinstance(object, Wrapper) or object is self._builtins or force:
oid = id(object)
self._pmemo[oid] = object
return oid
@@ -103,16 +154,103 @@
return None
def save_type(self, atype):
- self.save_reduce(TypeWrapper(atype, self._module),
- (atype.__bases__, atype.__dict__))
+ if atype.__module__ == "__builtin__":
+ self.save_global(atype)
+ else:
+ self.save_reduce(self.wrap(TypeWrapper, atype),
+ (atype.__bases__, atype.__dict__))
dispatch[TypeType] = save_type
+ dispatch[ClassType] = save_type
def save_function(self, func):
- self.save_reduce(FunctionWrapper(func, self._module),
+ self.save_reduce(self.wrap(FunctionWrapper, func),
(func.func_defaults, func.func_dict))
dispatch[FunctionType] = save_function
+
+ # New-style classes don't have real dicts. They have dictproxies.
+ # There's no official way to spell the dictproxy type, so we have
+ # to get it by using type() on an example.
+ dispatch[type(Wrapper.__dict__)] = pickle.Pickler.save_dict
+
+ def save(self, object, ignore=None):
+ # Override the save() implementation from pickle.py, because
+ # we don't ever want to invoke __reduce__() on builtin types
+ # that aren't picklable. Instead, we'd like to pickle all of
+ # those objects using the persistent_id() mechanism. There's
+ # no need to cover every type with this pickler, because it
+ # isn't being used for persistent just to create a copy.
+
+ # The ignored parameter is for compatible with Python 2.2,
+ # which has the old inst_persistent_id feature.
+ pid = self.persistent_id(object)
+ if pid is not None:
+ self.save_pers(pid)
+ return
+
+ d = id(object)
+ t = type(object)
+ if (t is TupleType) and (len(object) == 0):
+ if self.bin:
+ self.save_empty_tuple(object)
+ else:
+ self.save_tuple(object)
+ return
+
+ if d in self.memo:
+ self.write(self.get(self.memo[d][0]))
+ return
+
+ try:
+ f = self.dispatch[t]
+ except KeyError:
+ try:
+ issc = issubclass(t, TypeType)
+ except TypeError: # t is not a class
+ issc = 0
+ if issc:
+ self.save_global(object)
+ return
+
+ try:
+ reduce = dispatch_table[t]
+ except KeyError:
+ self.save_pers(self.persistent_id(object, True))
+ return
+ else:
+ tup = reduce(object)
+
+ if type(tup) is StringType:
+ self.save_global(object, tup)
+ return
+ if type(tup) is not TupleType:
+ raise PicklingError, "Value returned by %s must be a " \
+ "tuple" % reduce
+
+ l = len(tup)
+ if (l != 2) and (l != 3):
+ raise PicklingError, "tuple returned by %s must contain " \
+ "only two or three elements" % reduce
+
+ callable = tup[0]
+ arg_tup = tup[1]
+ if l > 2:
+ state = tup[2]
+ else:
+ state = None
+
+ if type(arg_tup) is not TupleType and arg_tup is not None:
+ raise PicklingError, "Second element of tuple returned " \
+ "by %s must be a tuple" % reduce
+
+ self.save_reduce(callable, arg_tup, state)
+ memo_len = len(self.memo)
+ self.write(self.put(memo_len))
+ self.memo[d] = (memo_len, object)
+ return
+
+ f(self, object)
class Unpickler(pickle.Unpickler):
@@ -123,26 +261,84 @@
def persistent_load(self, oid):
return self._pmemo[oid]
-def convert(module):
- """Return a copy of module dict with object conversions performed."""
- f = StringIO()
- memo = {}
- p = Pickler(f, module, memo)
- p.dump(module.__dict__)
- f.seek(0, 1) # reset file
- u = Unpickler(
-
-def update(newdict, olddict):
- """Modify newdict to preserve identity of persistent objects in olddict.
+class NameFinder:
+ """Find a canonical name for each update-able object."""
- If a persistent object is named in olddict and newdict, then
- persistent object identity should be preserved. The object should
- have the same oid in both dicts, and the object in the newdict
- should have the new state.
+ # XXX should we try to handle descriptors? If it looks like a
+ # descriptor, try calling it and passing the class object?
+
+ classTypes = {
+ TypeType: True,
+ ClassType: True,
+ PersistentClassMetaClass: True,
+ }
+
+ types = {
+ FunctionType: True,
+ PersistentFunction: True,
+ PersistentDescriptor: True,
+ }
+ types.update(classTypes)
+
+ def __init__(self, module):
+ self._names = {} # map object ids to (canonical name, obj) pairs
+ self.walkModule(module)
+
+ def names(self):
+ return [n for n, o in self._names.itervalues()]
+
+ def _walk(self, obj, name, fmt):
+ classes = []
+ for k, v in obj.__dict__.items():
+ aType = type(v)
+ anId = id(v)
+ if aType in self.types and not anId in self._names:
+ self._names[anId] = fmt % (name, k), v
+ if aType in self.classTypes:
+ classes.append((v, k))
+ for _klass, _name in classes:
+ self.walkClass(_klass, fmt % (name, _name))
+
+ def walkModule(self, mod):
+ self._walk(mod, "", "%s%s")
+
+ def walkClass(self, klass, name):
+ self._walk(klass, name, "%s.%s")
+
+ def replacements(self, aFinder):
+ """Return a dictionary of replacements.
+
+ self and aFinder are two NameFinder instances. Return a dict
+ of all the objects in the two that share the same name. The
+ keys are the ids in self and the values are the objects in
+ aFinder.
+ """
+ temp = {}
+ result = {}
+ for anId, (name, obj) in self._names.iteritems():
+ temp[name] = anId
+ for anId, (name, obj) in aFinder._names.iteritems():
+ if name in temp:
+ result[temp[name]] = obj
+ return result
+
+def convert(module, replacements):
+ """Convert object to persistent objects in module.
+
+ Use replacements dictionary to determine which objects to update
+ in place.
"""
+ print "convert", module, replacements
+ f = StringIO()
+ memo = {}
+ p = Pickler(f, module, memo, replacements)
+ moddict = module.__dict__
+ p.dump(moddict)
+ f.reset()
+ u = Unpickler(f, memo)
+ newdict = u.load()
+ module.__dict__.clear()
+ module.__dict__.update(newdict)
-def fixup(module):
- olddict = module.__dict__
- newdict = convert(olddict)
- update(newdict, olddict)
- module.__dict__ = newdict
+if __name__ == "__main__":
+ pass