[Checkins] SVN: zc.fsutils/branches/dev/ initial
Jim Fulton
jim at zope.com
Tue Sep 25 18:40:24 EDT 2007
Log message for revision 80062:
initial
Changed:
_U zc.fsutils/branches/dev/
A zc.fsutils/branches/dev/buildout.cfg
A zc.fsutils/branches/dev/setup.py
A zc.fsutils/branches/dev/src/
A zc.fsutils/branches/dev/src/zc/
A zc.fsutils/branches/dev/src/zc/__init__.py
A zc.fsutils/branches/dev/src/zc/fsutil/
A zc.fsutils/branches/dev/src/zc/fsutil/__init__.py
A zc.fsutils/branches/dev/src/zc/fsutil/references.py
A zc.fsutils/branches/dev/src/zc/fsutil/references.txt
A zc.fsutils/branches/dev/src/zc/fsutil/tests.py
-=-
Property changes on: zc.fsutils/branches/dev
___________________________________________________________________
Name: svn:ignore
+ develop-eggs
bin
parts
Added: zc.fsutils/branches/dev/buildout.cfg
===================================================================
--- zc.fsutils/branches/dev/buildout.cfg (rev 0)
+++ zc.fsutils/branches/dev/buildout.cfg 2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,12 @@
+[buildout]
+parts = script
+develop = .
+
+[script]
+recipe = zc.recipe.egg
+eggs = ${test:eggs}
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = zc.fsutil
+
Property changes on: zc.fsutils/branches/dev/buildout.cfg
___________________________________________________________________
Name: svn:eol-style
+ native
Added: zc.fsutils/branches/dev/setup.py
===================================================================
--- zc.fsutils/branches/dev/setup.py (rev 0)
+++ zc.fsutils/branches/dev/setup.py 2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,24 @@
+from setuptools import setup, find_packages
+
+entry_points="""
+[console_scripts]
+references = zc.fsutil.references:references_script
+
+"""
+
+setup(
+ name = "zc.fsutil",
+ description = "ZODB Database Utilities",
+ version = ".1",
+ license = "ZPL",
+ packages = find_packages('src'),
+ include_package_data = True,
+ zip_safe = False,
+ package_dir = {'':'src'},
+ namespace_packages = ['zc'],
+ entry_points = entry_points,
+ install_requires = [
+ 'setuptools',
+ 'ZODB3',
+ ],
+ )
Property changes on: zc.fsutils/branches/dev/setup.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zc.fsutils/branches/dev/src/zc/__init__.py
===================================================================
--- zc.fsutils/branches/dev/src/zc/__init__.py (rev 0)
+++ zc.fsutils/branches/dev/src/zc/__init__.py 2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
Property changes on: zc.fsutils/branches/dev/src/zc/__init__.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zc.fsutils/branches/dev/src/zc/fsutil/__init__.py
===================================================================
--- zc.fsutils/branches/dev/src/zc/fsutil/__init__.py (rev 0)
+++ zc.fsutils/branches/dev/src/zc/fsutil/__init__.py 2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1 @@
+#
Property changes on: zc.fsutils/branches/dev/src/zc/fsutil/__init__.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zc.fsutils/branches/dev/src/zc/fsutil/references.py
===================================================================
--- zc.fsutils/branches/dev/src/zc/fsutil/references.py (rev 0)
+++ zc.fsutils/branches/dev/src/zc/fsutil/references.py 2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,103 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Analyze database references
+"""
+
+import cPickle, cStringIO, sys
+
+import ZODB.FileStorage.FileStorage
+import ZODB.utils
+
+
+def references(iterator):
+ """Create a database of database references.
+
+ Return a dictionary mapping oids to dictionaries with keys 'from'
+ and 'to' with values that are dictionaries mapping oids to lists
+ of references.
+ """
+ result = {}
+ for trans in iterator:
+ for record in trans:
+ from_oid = ZODB.utils.oid_repr(record.oid)
+ from_data = result.get(from_oid)
+ if from_data is None:
+ from_data = result[from_oid] = {
+ 'from': {}, 'to': {}, 'serials': [],
+ }
+ from_data['serials'].append(record.tid)
+
+ refs = []
+ u = cPickle.Unpickler(cStringIO.StringIO(record.data))
+ u.persistent_load = refs
+ u.noload()
+ u.noload()
+ for ref in refs:
+ if isinstance(ref, tuple):
+ to_oid = ZODB.utils.oid_repr(ref[0])
+ elif isinstance(ref, str):
+ to_oid = ZODB.utils.oid_repr(ref)
+ elif isinstance(ref, list):
+ if len(ref) == 1:
+ to_oid = ZODB.utils.oid_repr(ref[0])
+ else:
+ try:
+ reference_type, args = ref
+ except ValueError:
+ print 'wtf', ref
+ continue
+
+ if reference_type == 'w':
+ to_oid = ZODB.utils.oid_repr(args[0])
+ elif reference_type in 'nm':
+ to_oid = args[0], ZODB.utils.oid_repr(args[1])
+ else:
+ print wtf, reference_type, args
+ else:
+ print 'wtf', ref
+ continue
+
+ ref = dict(ref=ref, tid=trans.tid, tpos=trans._tpos)
+
+ from_to = from_data['to'].get(to_oid)
+ if from_to is None:
+ from_to = from_data['to'][to_oid] = []
+ from_to.append(ref)
+
+ to_data = result.get(to_oid)
+ if to_data is None:
+ to_data = result[to_oid] = {
+ 'to': {}, 'from': {}, 'serials': [],
+ }
+ to_from = to_data['from'].get(from_oid)
+ if to_from is None:
+ to_from = to_data['from'][from_oid] = []
+ to_from.append(ref)
+
+ return result
+
+def references_script(args=None):
+ if args is None:
+ args = sys.argv[1:]
+
+ [inp, outp] = args
+
+ iterator = sys.modules['ZODB.FileStorage.FileStorage' # :(
+ ].FileIterator(inp)
+ data = references(iterator)
+ cPickle.Pickler(open(outp, 'w'), 1).dump(data)
+
+
+
+
Property changes on: zc.fsutils/branches/dev/src/zc/fsutil/references.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zc.fsutils/branches/dev/src/zc/fsutil/references.txt
===================================================================
--- zc.fsutils/branches/dev/src/zc/fsutil/references.txt (rev 0)
+++ zc.fsutils/branches/dev/src/zc/fsutil/references.txt 2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,233 @@
+References analysis utility
+===========================
+
+The references module provides functions and scripts for analyzing
+database references.
+
+Let's create a couple of databases.
+
+ >>> from ZODB.FileStorage import FileStorage
+ >>> fs1 = FileStorage('fs1')
+ >>> fs2 = FileStorage('fs2')
+ >>> databases = {}
+ >>> from ZODB.DB import DB
+ >>> db1 = DB(fs1, databases=databases, database_name='db1')
+ >>> db2 = DB(fs2, databases=databases, database_name='db2')
+ >>> c1 = db1.open()
+ >>> c2 = c1.get_connection('db2')
+ >>> from ZODB.tests.util import P
+
+We create a simple chain of objects in each db.
+
+ >>> c1.root()['p'] = P('1.p')
+ >>> c1.root()['p'].p = P('1.p.p')
+ >>> c2.root()['p'] = P('2.p')
+ >>> c2.root()['p'].p = P('2.p.p')
+ >>> from transaction import commit
+ >>> commit()
+
+We create a cross database reference:
+
+ >>> c1.root()['p'].x = c2.root()['p'].p
+ >>> commit()
+
+Let's create a weak reference too:
+
+ >>> import persistent.wref
+ >>> c2.root()['w'] = persistent.wref.WeakRef(c2.root()['p'].p)
+ >>> commit()
+
+Now, let's be a bit abusive. First, we'll delete the object we
+created the references to. First, we'll create a Python reference to
+it.
+
+ >>> ob = c2.root()['p'].p
+ >>> del c2.root()['p'].p
+ >>> commit()
+
+Now, we'll pack the second database:
+
+ >>> from ZODB.tests.util import pack
+ >>> pack(db2)
+
+and be evil by putting the object back. :)
+
+ >>> c2.root()['p'].p = ob
+ >>> del ob
+ >>> commit()
+
+ >>> c1.cacheMinimize()
+ >>> c2.cacheMinimize()
+
+Now, if we try to access the deleted object, we'll get a POSKeyError:
+
+ >>> c2.root()['p'].p
+ Traceback (most recent call last):
+ ...
+ POSKeyError: 0x02
+
+ >>> c1.root()['p'].x
+ Traceback (most recent call last):
+ ...
+ POSKeyError: 0x02
+
+ >>> c2.root()['w']()
+ Traceback (most recent call last):
+ ...
+ POSKeyError: 0x02
+
+OK, so we got these errors. We know why because we created them on
+purpose. In practice, this isn't the case. The references module can
+help us figure out what's going on.
+
+ >>> import zc.fsutil.references
+
+The references function computes a data structure from a storage
+iterator:
+
+ >>> refs1 = zc.fsutil.references.references(fs1.iterator())
+
+This data structure has information about all of the object ids that
+either had data or were references.
+
+ >>> from pprint import pprint
+ >>> pprint(refs1, width=1)
+ {'0x00': {'from': {},
+ 'serials': ['\x03p\x98[\xe0C\xd73',
+ '\x03p\x98[\xe0C\xd74'],
+ 'to': {'0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x01',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd74',
+ 'tpos': 168L}]}},
+ '0x01': {'from': {'0x00': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x01',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd74',
+ 'tpos': 168L}]},
+ 'serials': ['\x03p\x98[\xe0C\xd74',
+ '\x03p\x98[\xe0C\xd75'],
+ 'to': {'0x02': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd74',
+ 'tpos': 168L},
+ {'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 535L}],
+ ('db2', '0x02'): [{'ref': ['m',
+ ('db2',
+ '\x00\x00\x00\x00\x00\x00\x00\x02',
+ None)],
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 535L}]}},
+ '0x02': {'from': {'0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd74',
+ 'tpos': 168L},
+ {'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 535L}]},
+ 'serials': ['\x03p\x98[\xe0C\xd74'],
+ 'to': {}},
+ ('db2', '0x02'): {'from': {'0x01': [{'ref': ['m',
+ ('db2',
+ '\x00\x00\x00\x00\x00\x00\x00\x02',
+ None)],
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 535L}]},
+ 'serials': [],
+ 'to': {}}}
+
+There's an entry for each object referenced. Each entry is a
+dictionary with 3 keys:
+
+from
+ Is a dictionary containing information about references to the entry's
+ object id. Each entry has as it's key, the refering object id and
+ as it's values, the list of references from that object id.
+
+to
+ Is a dictionary containing information about references from the
+ entry's object id. Each entry has as it's key, the object id of the
+ object being referenced, and as it's values, the list of references
+ to that object id.
+
+serials
+ Is a list of serial ids (transaction ids) for the entry's object.
+
+ An entry with empty serials is a missing object. It is refered to,
+ but there are no records for it.
+
+Note that object 1 is refered to by object 0 and refers to object 2
+and to object 2 in the second database. It has 2 database records,
+indicated by 2 values in it's serials.
+
+The references_script function is intended to be used as a setuptools
+entry point. We'll call it directly, passing command line arguments,
+which are the name of an input file and the name of an output data file:
+
+ >>> zc.fsutil.references.references_script(['fs2', 'fs2.dat'])
+
+The data file is just a pickle file:
+
+ >>> import cPickle
+ >>> refs2 = cPickle.Unpickler(open('fs2.dat')).load()
+ >>> pprint(refs2, width=1)
+ {'0x00': {'from': {},
+ 'serials': ['\x03p\x98[\xe0C\xd75'],
+ 'to': {'0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x01',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 4L}],
+ '0x02': [{'ref': ['w',
+ ('\x00\x00\x00\x00\x00\x00\x00\x02',)],
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 4L}]}},
+ '0x01': {'from': {'0x00': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x01',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 4L}]},
+ 'serials': ['\x03p\x98[\xe0C\xd76',
+ '\x03p\x98[\xe0C\xd77'],
+ 'to': {'0x02': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd77',
+ 'tpos': 324L}]}},
+ '0x02': {'from': {'0x00': [{'ref': ['w',
+ ('\x00\x00\x00\x00\x00\x00\x00\x02',)],
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 4L}],
+ '0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd77',
+ 'tpos': 324L}]},
+ 'serials': [],
+ 'to': {}}}
+
+In database 2, we see that object id is missing because it doesn't
+have any serials.
+
+We can query these data structures using Python. For example, to dind
+missing objects (and the in-database objects that reference them:
+
+ >>> pprint([(oid, data) for (oid, data) in refs2.iteritems()
+ ... if not data['serials']
+ ... ], width=1)
+ [('0x02',
+ {'from': {'0x00': [{'ref': ['w',
+ ('\x00\x00\x00\x00\x00\x00\x00\x02',)],
+ 'tid': '\x03p\x98[\xe0C\xd75',
+ 'tpos': 4L}],
+ '0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+ None),
+ 'tid': '\x03p\x98[\xe0C\xd77',
+ 'tpos': 324L}]},
+ 'serials': [],
+ 'to': {}})]
+
+Here we see the broken weak reference from object 0 and the broken
+ordinary reference from object 1.
+
+Our data structure can't tell us about broken cross-database references
+directly.
+
Property changes on: zc.fsutils/branches/dev/src/zc/fsutil/references.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Added: zc.fsutils/branches/dev/src/zc/fsutil/tests.py
===================================================================
--- zc.fsutils/branches/dev/src/zc/fsutil/tests.py (rev 0)
+++ zc.fsutils/branches/dev/src/zc/fsutil/tests.py 2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,38 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import time, unittest
+from zope.testing import doctest
+import zope.testing.setupstack
+
+
+
+def setUp(test):
+ zope.testing.setupstack.setUpDirectory(test)
+ orig_time = time.time
+ def restore_time_time():
+ time.time = orig_time
+ zope.testing.setupstack.register(test, restore_time_time)
+ time.time = lambda : 1190753032.5621099
+
+
+
+
+def test_suite():
+ return unittest.TestSuite((
+ doctest.DocFileSuite(
+ 'references.txt',
+ setUp=setUp, tearDown=zope.testing.setupstack.tearDown,
+ ),
+ ))
+
Property changes on: zc.fsutils/branches/dev/src/zc/fsutil/tests.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
More information about the Checkins
mailing list