[Checkins] SVN: zodbupdate/trunk/ Fix pickling error when there is missing factories.
Sylvain Viollon
sylvain at infrae.com
Wed Oct 6 13:38:43 EDT 2010
Log message for revision 117308:
Fix pickling error when there is missing factories.
Changed:
U zodbupdate/trunk/CHANGES.txt
U zodbupdate/trunk/src/zodbupdate/serialize.py
U zodbupdate/trunk/src/zodbupdate/tests.py
U zodbupdate/trunk/src/zodbupdate/update.py
-=-
Modified: zodbupdate/trunk/CHANGES.txt
===================================================================
--- zodbupdate/trunk/CHANGES.txt 2010-10-06 15:12:11 UTC (rev 117307)
+++ zodbupdate/trunk/CHANGES.txt 2010-10-06 17:38:43 UTC (rev 117308)
@@ -5,8 +5,19 @@
----------------
- More debug logging shows now the currently processed OID
- (that is helpful to determine which object misses the factory)
+ (that is helpful to determine which object misses the factory).
+- Support for missing factories have been improved: an error used to
+ occur if a pickle needed an update and contained a reference to a
+ missing class (not instance of this class).
+
+ This case is now fixed.
+
+- Python 2.4 is no longer supported. Please stick to version 0.3 if
+ you need Python .4 support.
+
+
+
0.4 (2010-07-14)
----------------
Modified: zodbupdate/trunk/src/zodbupdate/serialize.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/serialize.py 2010-10-06 15:12:11 UTC (rev 117307)
+++ zodbupdate/trunk/src/zodbupdate/serialize.py 2010-10-06 17:38:43 UTC (rev 117308)
@@ -16,17 +16,77 @@
import cStringIO
import logging
import types
+import sys
from ZODB.broken import find_global, Broken, rebuild
from zodbupdate import utils
logger = logging.getLogger('zodbupdate')
+known_broken_modules = {}
-def isbroken(symb):
+def is_broken(symb):
+ """Return true if the given symbol is broken.
+ """
return isinstance(symb, types.TypeType) and Broken in symb.__mro__
+def create_broken_module_for(symb):
+ """If your pickle refer a broken class (not an instance of it, a
+ reference to the class symbol itself) you have no choice than
+ having this module available in the same module and with the
+ same name, otherwise repickling doesn't work (as both pickle
+ and cPikle __import__ the module, and verify the class symbol
+ is the same).
+ """
+ parts = symb.__module__.split('.')
+ previous_module = None
+ previous_name = None
+ for fullname, name in reversed(
+ [('.'.join(parts[0:p+1]), parts[p]) for p in range(1, len(parts))]):
+ if fullname not in sys.modules:
+ if fullname not in known_broken_modules:
+ module = types.ModuleType(fullname)
+ module.__name__ = name
+ module.__file__ = '<broken module to pickle class reference>'
+ module.__path__ = []
+ known_broken_modules[fullname] = module
+ else:
+ module = known_broken_modules[fullname]
+ if previous_module and previous_name:
+ setattr(module, previous_name, previous_module)
+ previous_module = module
+ previous_name = name
+ else:
+ if previous_module and previous_name:
+ setattr(sys.modules[fullname], previous_name, previous_module)
+ break
+ if symb.__module__ in known_broken_modules:
+ setattr(known_broken_modules[symb.__module__], symb.__name__, symb)
+ else:
+ setattr(sys.modules[symb.__module__], symb.__name__, symb)
+
+
+class BrokenModuleFinder(object):
+ """This broken module finder works with create_broken_module_for.
+ """
+
+ def load_module(self, fullname):
+ module = known_broken_modules[fullname]
+ if fullname not in sys.modules:
+ sys.modules[fullname] = module
+ module.__loader__ = self
+ return module
+
+ def find_module(self, fullname, path=None):
+ if fullname in known_broken_modules:
+ return self
+ return None
+
+
+sys.meta_path.append(BrokenModuleFinder())
+
+
class NullIterator(object):
"""An empty iterator that doesn't gives any result.
"""
@@ -102,9 +162,10 @@
return self.__changes[symb_info]
else:
symb = find_global(*symb_info, Broken=ZODBBroken)
- if isbroken(symb):
- logger.warning(u'Warning: Missing factory for %s' %
- u' '.join(symb_info))
+ if is_broken(symb):
+ logger.warning(
+ u'Warning: Missing factory for %s' % u' '.join(symb_info))
+ create_broken_module_for(symb)
elif hasattr(symb, '__name__') and hasattr(symb, '__module__'):
new_symb_info = (symb.__module__, symb.__name__)
if new_symb_info != symb_info:
@@ -174,10 +235,10 @@
"""
if isinstance(class_meta, tuple):
symb, args = class_meta
- if isbroken(symb):
+ if is_broken(symb):
symb_info = (symb.__module__, symb.__name__)
- logger.warning(u'Warning: Missing factory for %s' %
- u' '.join(symb_info))
+ logger.warning(
+ u'Warning: Missing factory for %s' % u' '.join(symb_info))
return (symb_info, args)
elif isinstance(symb, tuple):
return self.__update_symb(symb), args
@@ -207,9 +268,9 @@
try:
pickler.dump(class_meta)
pickler.dump(data)
- except cPickle.PicklingError:
- # Could not pickle that record, likely due to a broken
- # class ignore it.
+ except cPickle.PicklingError, error:
+ logger.error('Error while pickling modified record: %s' % error)
+ # Could not pickle that record, skip it.
return None
output_file.truncate()
Modified: zodbupdate/trunk/src/zodbupdate/tests.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/tests.py 2010-10-06 15:12:11 UTC (rev 117307)
+++ zodbupdate/trunk/src/zodbupdate/tests.py 2010-10-06 17:38:43 UTC (rev 117308)
@@ -15,10 +15,8 @@
import ZODB
import ZODB.broken
import ZODB.FileStorage
-import logging
import os
import persistent
-import pickle
import sys
import tempfile
import transaction
Modified: zodbupdate/trunk/src/zodbupdate/update.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/update.py 2010-10-06 15:12:11 UTC (rev 117307)
+++ zodbupdate/trunk/src/zodbupdate/update.py 2010-10-06 17:38:43 UTC (rev 117308)
@@ -12,7 +12,6 @@
#
##############################################################################
-from ZODB.DB import DB
from ZODB.FileStorage import FileStorage
from struct import pack, unpack
import ZODB.POSException
@@ -20,9 +19,6 @@
import ZODB.utils
import cStringIO
import logging
-import pickle
-import pickletools
-import sys
import transaction
import zodbupdate.serialize
More information about the checkins
mailing list