[Zope-CVS] CVS: Products/AdaptableStorage/zodb - RemainingState.py:1.1 ASConnection.py:1.8 public.py:1.2
Shane Hathaway
shane@zope.com
Fri, 13 Dec 2002 15:42:04 -0500
Update of /cvs-repository/Products/AdaptableStorage/zodb
In directory cvs.zope.org:/tmp/cvs-serv32288/zodb
Modified Files:
ASConnection.py public.py
Added Files:
RemainingState.py
Log Message:
Hopefully solved the problem of unmanaged persistent objects.
Because AdaptableStorage lets the mappers choose their own persistence
boundaries, some objects that inherit from Persistent may be unknown
to ZODB, even though they are in the database. I call these unmanaged
persistent objects.
The problem with unmanaged persistent objects surfaces when you change
them without changing their container. For example, if you add an
item to a BTree, the BTree object (which is managed) does not get
changed, but rather a contained Bucket object (which is often
unmanaged). We really need the BTree to be notified when the Buckets
change.
To solve this, certain aspect serializers (currently only
RemainderSerializer) now detect unmanaged persistent objects and addq
them to a list. ZODB looks over this list and assigns them a one-off
"_p_jar" with a register() method. This special register() method
just sets the _p_changed attribute of the managed object, notifying
ZODB that it must be saved.
I think this is the best solution, even though it's awfully complex to
explain. The change involved moving the RemainingState aspect to the
"zodb" subpackage, since it already depended on the particulars of the
Persistent base class anyway. It also required changing
ObjectSerializer so that the event object gets constructed by the
caller, which is appropriate, I think.
Also made some minor changes.
=== Added File Products/AdaptableStorage/zodb/RemainingState.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Aspect for (de)serializing the remaining state of a Persistent object
$Id: RemainingState.py,v 1.1 2002/12/13 20:42:03 shane Exp $
"""
from cStringIO import StringIO
from cPickle import Pickler, Unpickler
from types import DictType
from ZODB import Persistent
from serial_public import \
IAspectSerializer, FieldSchema, \
IFullSerializationEvent, IFullDeserializationEvent
class RemainingState:
__implements__ = IAspectSerializer
schema = FieldSchema('data', 'string')
def getSchema(self):
return self.schema
def serialize(self, object, event):
assert IFullSerializationEvent.isImplementedBy(event)
assert isinstance(object, Persistent)
state = object.__dict__.copy()
for key in state.keys():
if key.startswith('_v_'):
del state[key]
for attrname in event.getSerializedAttributeNames():
if state.has_key(attrname):
del state[attrname]
if not state:
# No data needs to be stored
return ''
outfile = StringIO()
p = Pickler(outfile)
unmanaged = []
def persistent_id(ob, getInternalRef=event.getInternalRef,
unmanaged=unmanaged):
ref = getInternalRef(ob)
if ref is None:
if isinstance(ob, Persistent):
# Persistent objects that end up in the remainder
# are unmanaged. Tell ZODB about them so that
# ZODB can deal with them specially.
unmanaged.append(ob)
return ref
p.persistent_id = persistent_id
p.dump(state)
p.dump(unmanaged)
s = outfile.getvalue()
event.addUnmanagedPersistentObjects(unmanaged)
return s
def deserialize(self, object, event, state):
assert IFullDeserializationEvent.isImplementedBy(event)
assert isinstance(object, Persistent)
if state:
infile = StringIO(state)
u = Unpickler(infile)
u.persistent_load = event.loadInternalRef
s = u.load()
object.__dict__.update(s)
try:
unmanaged = u.load()
except EOFError:
# old pickle
pass
else:
event.addUnmanagedPersistentObjects(unmanaged)
=== Products/AdaptableStorage/zodb/ASConnection.py 1.7 => 1.8 ===
--- Products/AdaptableStorage/zodb/ASConnection.py:1.7 Mon Dec 9 17:11:08 2002
+++ Products/AdaptableStorage/zodb/ASConnection.py Fri Dec 13 15:42:03 2002
@@ -20,12 +20,13 @@
from time import time
from types import StringType, TupleType
-import ZODB
+from ZODB import Persistent
from ZODB.Connection import Connection, StringIO, Unpickler, Pickler, \
ConflictError, ReadConflictError, ExtensionKlass, LOG, ERROR
from consts import SERIAL0, DEBUG
-from serial_public import IKeyedObjectSystem
+from serial_public import IKeyedObjectSystem, SerializationEvent, \
+ DeserializationEvent
class ASConnection (Connection):
@@ -247,7 +248,9 @@
ser = mapper.getSerializer()
if DEBUG:
print 'serializing', repr(oid), repr(serial)
- state, ext_refs = ser.serialize(self, mapper, keychain, object)
+ event = SerializationEvent(self, mapper, keychain, object)
+ state = ser.serialize(object, event)
+ ext_refs = event.getExternalRefs()
if ext_refs:
for (ext_keychain, ext_ref) in ext_refs:
if (not ext_ref._p_serial
@@ -265,6 +268,10 @@
ext_ref._p_oid = ext_oid
stack.append(ext_ref)
+ unmanaged = event.getUnmanagedPersistentObjects()
+ if unmanaged:
+ self.handleUnmanaged(object, unmanaged)
+
seek(0)
clear_memo()
dump((classification, mapper_names))
@@ -339,7 +346,12 @@
ser = mapper.getSerializer()
if DEBUG:
print 'deserializing', repr(oid), repr(serial)
- ser.deserialize(self, mapper, keychain, object, state)
+ event = DeserializationEvent(self, mapper, keychain, object)
+ ser.deserialize(object, event, state)
+
+ unmanaged = event.getUnmanagedPersistentObjects()
+ if unmanaged:
+ self.handleUnmanaged(object, unmanaged)
if mapper.isVolatile():
v = self._volatile
@@ -374,6 +386,17 @@
return '<%s at %08x%s>' % (self.__class__.__name__, id(self), ver)
+ def handleUnmanaged(self, object, unmanaged):
+ for o in unmanaged:
+ if isinstance(o, Persistent):
+ if o._p_jar is None:
+ o._p_oid = 'unmanaged'
+ o._p_jar = UnmanagedJar(self, object._p_oid)
+ else:
+ # Turn off the "changed" flag
+ o._p_changed = 0
+
+
# IKeyedObjectSystem implementation
def loadStub(self, keychain, mapper_names=None):
@@ -419,4 +442,26 @@
def sync(self):
self.invalidateVolatileObjects()
Connection.sync(self)
+
+
+
+class UnmanagedJar:
+ """Special jar for unmanaged persistent objects.
+
+ There is one such jar for each unmanaged persistent object. All
+ it does is notify the managed persistent object of changes.
+
+ Note that unmanaged persistent objects should never be ghosted!
+ Instead, when the managed persistent object gets ghosted, it
+ usually removes the last reference to the unmanaged object, which
+ is then deallocated.
+ """
+
+ def __init__(self, real_jar, real_oid):
+ self.real_jar = real_jar
+ self.real_oid = real_oid
+
+ def register(self, ob):
+ o = self.real_jar[self.real_oid]
+ o._p_changed = 1
=== Products/AdaptableStorage/zodb/public.py 1.1 => 1.2 ===
--- Products/AdaptableStorage/zodb/public.py:1.1 Tue Dec 3 18:10:55 2002
+++ Products/AdaptableStorage/zodb/public.py Fri Dec 13 15:42:03 2002
@@ -22,3 +22,4 @@
from ASStorage import ASStorage
from OIDEncoder import OIDEncoder
from StaticResource import StaticResource
+from RemainingState import RemainingState