[Zodb-checkins] SVN: ZODB/trunk/src/Z Added support for wrapper storages that transform pickle data. (I
Jim Fulton
jim at zope.com
Tue May 18 11:22:14 EDT 2010
Log message for revision 112462:
Added support for wrapper storages that transform pickle data. (I
thought this support already existed, but found out I was wrong when I
tried to make it work. :/) Applications for this include compression
and encryption. An example wrapper storage implementation,
ZODB.tests.hexstorage, was included for testing.
It is important that storage implementations not assume that
storages contain pickles. Renamed IStorageDB to IStorageWrapper and
expanded it to provide methods for transforming and untransforming
data records. Storages implementations should use these methods to
get pickle data from stored records.
Changed:
U ZODB/trunk/src/ZEO/StorageServer.py
U ZODB/trunk/src/ZEO/tests/IterationTests.py
U ZODB/trunk/src/ZEO/tests/testZEO.py
U ZODB/trunk/src/ZODB/ConflictResolution.py
U ZODB/trunk/src/ZODB/DB.py
U ZODB/trunk/src/ZODB/FileStorage/FileStorage.py
U ZODB/trunk/src/ZODB/FileStorage/fspack.py
U ZODB/trunk/src/ZODB/blob.py
U ZODB/trunk/src/ZODB/interfaces.py
U ZODB/trunk/src/ZODB/tests/BasicStorage.py
A ZODB/trunk/src/ZODB/tests/component.xml
A ZODB/trunk/src/ZODB/tests/hexstorage.py
U ZODB/trunk/src/ZODB/tests/testDemoStorage.py
U ZODB/trunk/src/ZODB/tests/testFileStorage.py
-=-
Modified: ZODB/trunk/src/ZEO/StorageServer.py
===================================================================
--- ZODB/trunk/src/ZEO/StorageServer.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZEO/StorageServer.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -778,6 +778,7 @@
def invalidateCache(self):
self.server._invalidateCache(self.storage_id)
+ transform_record_data = untransform_record_data = lambda self, data: data
class StorageServer:
Modified: ZODB/trunk/src/ZEO/tests/IterationTests.py
===================================================================
--- ZODB/trunk/src/ZEO/tests/IterationTests.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZEO/tests/IterationTests.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -35,7 +35,12 @@
def checkIteratorExhaustionStorage(self):
# Test the storage's garbage collection mechanism.
+ self._dostore()
iterator = self._storage.iterator()
+
+ # At this point, a wrapping iterator might not have called the CS
+ # iterator yet. We'll consume one item to make sure this happens.
+ iterator.next()
self.assertEquals(1, len(self._storage._iterator_ids))
iid = list(self._storage._iterator_ids)[0]
self.assertEquals([], list(iterator))
@@ -47,19 +52,26 @@
def checkIteratorGCSpanTransactions(self):
# Keep a hard reference to the iterator so it won't be automatically
# garbage collected at the transaction boundary.
+ self._dostore()
iterator = self._storage.iterator()
self._dostore()
# As the iterator was not garbage collected, we can still use it. (We
# don't see the transaction we just wrote being picked up, because
# iterators only see the state from the point in time when they were
# created.)
- self.assertEquals([], list(iterator))
+ self.assert_(list(iterator))
def checkIteratorGCStorageCommitting(self):
# We want the iterator to be garbage-collected, so we don't keep any
# hard references to it. The storage tracks its ID, though.
- self._storage.iterator()
+ # The odd little jig we do below arises from the fact that the
+ # CS iterator may not be constructed right away if the CS is wrapped.
+ # We need to actually do some iteration to get the iterator created.
+ # We do a store to make sure the iterator isn't exhausted right away.
+ self._dostore()
+ self._storage.iterator().next()
+
self.assertEquals(1, len(self._storage._iterator_ids))
iid = list(self._storage._iterator_ids)[0]
@@ -70,8 +82,15 @@
self.assertRaises(KeyError, self._storage._server.iterator_next, iid)
def checkIteratorGCStorageTPCAborting(self):
- self._storage.iterator()
+ # The odd little jig we do below arises from the fact that the
+ # CS iterator may not be constructed right away if the CS is wrapped.
+ # We need to actually do some iteration to get the iterator created.
+ # We do a store to make sure the iterator isn't exhausted right away.
+ self._dostore()
+ self._storage.iterator().next()
+
iid = list(self._storage._iterator_ids)[0]
+
t = transaction.Transaction()
self._storage.tpc_begin(t)
self._storage.tpc_abort(t)
@@ -79,7 +98,14 @@
self.assertRaises(KeyError, self._storage._server.iterator_next, iid)
def checkIteratorGCStorageDisconnect(self):
- self._storage.iterator()
+
+ # The odd little jig we do below arises from the fact that the
+ # CS iterator may not be constructed right away if the CS is wrapped.
+ # We need to actually do some iteration to get the iterator created.
+ # We do a store to make sure the iterator isn't exhausted right away.
+ self._dostore()
+ self._storage.iterator().next()
+
iid = list(self._storage._iterator_ids)[0]
t = transaction.Transaction()
self._storage.tpc_begin(t)
Modified: ZODB/trunk/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/trunk/src/ZEO/tests/testZEO.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZEO/tests/testZEO.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -47,6 +47,7 @@
import ZEO.zrpc.connection
import ZODB
import ZODB.blob
+import ZODB.tests.hexstorage
import ZODB.tests.testblob
import ZODB.tests.util
import ZODB.utils
@@ -59,6 +60,7 @@
pass
def invalidateCache(*unused):
pass
+ transform_record_data = untransform_record_data = lambda self, v: v
class CreativeGetState(persistent.Persistent):
@@ -94,7 +96,8 @@
def checkZEOInvalidation(self):
addr = self._storage._addr
- storage2 = ClientStorage(addr, wait=1, min_disconnect_poll=0.1)
+ storage2 = self._wrap_client(
+ ClientStorage(addr, wait=1, min_disconnect_poll=0.1))
try:
oid = self._storage.new_oid()
ob = MinPO('first')
@@ -196,13 +199,16 @@
self.blob_cache_dir = tempfile.mkdtemp(
'blob_cache',
dir=os.path.abspath(os.getcwd()))
- self._storage = ClientStorage(
+ self._storage = self._wrap_client(ClientStorage(
zport, '1', cache_size=20000000,
min_disconnect_poll=0.5, wait=1,
wait_timeout=60, blob_dir=self.blob_cache_dir,
- shared_blob_dir=self.shared_blob_dir)
+ shared_blob_dir=self.shared_blob_dir))
self._storage.registerDB(DummyDB())
+ def _wrap_client(self, client):
+ return client
+
def tearDown(self):
self._storage.close()
for server in self._servers:
@@ -317,6 +323,16 @@
</filestorage>
"""
+ _expected_interfaces = (
+ ('ZODB.interfaces', 'IStorageRestoreable'),
+ ('ZODB.interfaces', 'IStorageIteration'),
+ ('ZODB.interfaces', 'IStorageUndoable'),
+ ('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
+ ('ZODB.interfaces', 'IExternalGC'),
+ ('ZODB.interfaces', 'IStorage'),
+ ('zope.interface', 'Interface'),
+ )
+
def checkInterfaceFromRemoteStorage(self):
# ClientStorage itself doesn't implement IStorageIteration, but the
# FileStorage on the other end does, and thus the ClientStorage
@@ -326,19 +342,49 @@
self.failUnless(ZODB.interfaces.IStorageIteration.providedBy(
self._storage))
# This is communicated using ClientStorage's _info object:
- self.assertEquals(
- (('ZODB.interfaces', 'IStorageRestoreable'),
- ('ZODB.interfaces', 'IStorageIteration'),
- ('ZODB.interfaces', 'IStorageUndoable'),
- ('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
- ('ZODB.interfaces', 'IExternalGC'),
- ('ZODB.interfaces', 'IStorage'),
- ('zope.interface', 'Interface'),
- ),
+ self.assertEquals(self._expected_interfaces,
self._storage._info['interfaces']
)
+class FileStorageHexTests(FileStorageTests):
+ _expected_interfaces = (
+ ('ZODB.interfaces', 'IStorageRestoreable'),
+ ('ZODB.interfaces', 'IStorageIteration'),
+ ('ZODB.interfaces', 'IStorageUndoable'),
+ ('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
+ ('ZODB.interfaces', 'IExternalGC'),
+ ('ZODB.interfaces', 'IStorage'),
+ ('ZODB.interfaces', 'IStorageWrapper'),
+ ('zope.interface', 'Interface'),
+ )
+ def getConfig(self):
+ return """\
+ %import ZODB.tests
+ <hexstorage>
+ <filestorage 1>
+ path Data.fs
+ </filestorage>
+ </hexstorage>
+ """
+
+class FileStorageClientHexTests(FileStorageHexTests):
+
+ def getConfig(self):
+ return """\
+ %import ZODB.tests
+ <serverhexstorage>
+ <filestorage 1>
+ path Data.fs
+ </filestorage>
+ </serverhexstorage>
+ """
+
+ def _wrap_client(self, client):
+ return ZODB.tests.hexstorage.HexStorage(client)
+
+
+
class MappingStorageTests(GenericTests):
"""ZEO backed by a Mapping storage."""
@@ -1405,7 +1451,8 @@
slow_test_classes = [
BlobAdaptedFileStorageTests, BlobWritableCacheTests,
- DemoStorageTests, FileStorageTests, MappingStorageTests,
+ MappingStorageTests, DemoStorageTests,
+ FileStorageTests, FileStorageHexTests, FileStorageClientHexTests,
]
quick_test_classes = [
Modified: ZODB/trunk/src/ZODB/ConflictResolution.py
===================================================================
--- ZODB/trunk/src/ZODB/ConflictResolution.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/ConflictResolution.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -53,6 +53,7 @@
def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial)
+ p = self._crs_untransform_record_data(p)
file = StringIO(p)
unpickler = Unpickler(file)
unpickler.find_global = find_global
@@ -80,13 +81,13 @@
def __cmp__(other):
'''if other is equivalent reference, return 0; else raise ValueError.
-
+
Equivalent in this case means that oid and database_name are the same.
If either is a weak reference, we only support `is` equivalence, and
otherwise raise a ValueError even if the datbase_names and oids are
the same, rather than guess at the correct semantics.
-
+
It is impossible to sort reliably, since the actual persistent
class may have its own comparison, and we have no idea what it is.
We assert that it is reasonably safe to assume that an object is
@@ -135,7 +136,7 @@
def __cmp__(self, other):
if self is other or (
- isinstance(other, PersistentReference) and
+ isinstance(other, PersistentReference) and
self.oid == other.oid and
self.database_name == other.database_name and
not self.weak and
@@ -179,6 +180,7 @@
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
try:
prfactory = PersistentReferenceFactory()
+ newpickle = self._crs_untransform_record_data(newpickle)
file = StringIO(newpickle)
unpickler = Unpickler(file)
unpickler.find_global = find_global
@@ -194,7 +196,7 @@
newargs = ()
if klass in _unresolvable:
- return None
+ raise ConflictError
newstate = unpickler.load()
inst = klass.__new__(klass, *newargs)
@@ -203,7 +205,7 @@
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
- return None
+ raise ConflictError
old = state(self, oid, oldSerial, prfactory)
committed = state(self, oid, committedSerial, prfactory, committedData)
@@ -215,9 +217,9 @@
pickler.inst_persistent_id = persistent_id
pickler.dump(meta)
pickler.dump(resolved)
- return file.getvalue(1)
+ return self._crs_transform_record_data(file.getvalue(1))
except (ConflictError, BadClassName):
- return None
+ pass
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
@@ -225,9 +227,19 @@
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.error("Unexpected error", exc_info=True)
- return None
-class ConflictResolvingStorage:
+ raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
+ data=newpickle)
+
+class ConflictResolvingStorage(object):
"Mix-in class that provides conflict resolution handling for storages"
tryToResolveConflict = tryToResolveConflict
+
+ _crs_transform_record_data = _crs_untransform_record_data = (
+ lambda self, o: o)
+
+ def registerDB(self, wrapper):
+ self._crs_untransform_record_data = wrapper.untransform_record_data
+ self._crs_transform_record_data = wrapper.transform_record_data
+ super(ConflictResolvingStorage, self).registerDB(wrapper)
Modified: ZODB/trunk/src/ZODB/DB.py
===================================================================
--- ZODB/trunk/src/ZODB/DB.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/DB.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -707,6 +707,8 @@
"""
self._connectionMap(lambda c: c.invalidateCache())
+ transform_record_data = untransform_record_data = lambda self, data: data
+
def objectCount(self):
return len(self.storage)
Modified: ZODB/trunk/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/FileStorage/FileStorage.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/FileStorage/FileStorage.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -476,14 +476,8 @@
committed_tid = h.tid
if oldserial != committed_tid:
- rdata = self.tryToResolveConflict(oid, committed_tid,
+ data = self.tryToResolveConflict(oid, committed_tid,
oldserial, data)
- if rdata is None:
- raise POSException.ConflictError(
- oid=oid, serials=(committed_tid, oldserial),
- data=data)
- else:
- data = rdata
pos = self._pos
here = pos + self._tfile.tell() + self._thl
@@ -843,10 +837,11 @@
except KeyError:
# couldn't find oid; what's the real explanation for this?
raise UndoError("_loadBack() failed for %s", oid)
- data = self.tryToResolveConflict(oid, ctid, tid, bdata, cdata)
-
- if data:
+ try:
+ data = self.tryToResolveConflict(oid, ctid, tid, bdata, cdata)
return data, 0, ipos
+ except POSException.ConflictError:
+ pass
raise UndoError("Some data were modified by a later transaction", oid)
@@ -961,7 +956,7 @@
except ZODB.POSException.POSKeyError:
pass # It was removed, so no need to copy data
else:
- if ZODB.blob.is_blob_record(up):
+ if self.is_blob_record(up):
# We're undoing a blob modification operation.
# We have to copy the blob data
tmp = ZODB.utils.mktemp(dir=self.fshelper.temp_dir)
@@ -1768,7 +1763,7 @@
def next(self):
if self._file is None:
- raise ZODB.interfaces.StorageStopIteration()
+ raise StopIteration()
pos = self._pos
while True:
@@ -1857,7 +1852,7 @@
return result
self.close()
- raise ZODB.interfaces.StorageStopIteration()
+ raise StopIteration()
class TransactionRecord(BaseStorage.TransactionRecord):
@@ -1916,7 +1911,7 @@
return Record(h.oid, h.tid, data, prev_txn, pos)
- raise ZODB.interfaces.StorageStopIteration()
+ raise StopIteration()
class Record(BaseStorage.DataRecord):
Modified: ZODB/trunk/src/ZODB/FileStorage/fspack.py
===================================================================
--- ZODB/trunk/src/ZODB/FileStorage/fspack.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/FileStorage/fspack.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -30,7 +30,6 @@
import logging
import os
-import ZODB.blob
import ZODB.fsIndex
import ZODB.POSException
@@ -502,7 +501,7 @@
data = self._file.read(h.plen)
else:
data = self.fetchDataViaBackpointer(h.oid, h.back)
- if data and ZODB.blob.is_blob_record(data):
+ if data and self._storage.is_blob_record(data):
# We need to remove the blob record. Maybe we
# need to remove oid:
Modified: ZODB/trunk/src/ZODB/blob.py
===================================================================
--- ZODB/trunk/src/ZODB/blob.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/blob.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -617,30 +617,25 @@
"""
self.dirty_oids = []
+ def registerDB(self, db):
+ self.__untransform_record_data = db.untransform_record_data
+ try:
+ m = super(BlobStorageMixin, self).registerDB
+ except AttributeError:
+ pass
+ else:
+ m(db)
+
+ def __untransform_record_data(self, record):
+ return record
+
+ def is_blob_record(self, record):
+ if record:
+ return is_blob_record(self.__untransform_record_data(record))
+
def copyTransactionsFrom(self, other):
- for trans in other.iterator():
- self.tpc_begin(trans, trans.tid, trans.status)
- for record in trans:
- blobfilename = None
- if is_blob_record(record.data):
- try:
- blobfilename = other.loadBlob(record.oid, record.tid)
- except POSKeyError:
- pass
- if blobfilename is not None:
- fd, name = tempfile.mkstemp(
- suffix='.tmp', dir=self.fshelper.temp_dir)
- os.close(fd)
- utils.cp(open(blobfilename, 'rb'), open(name, 'wb'))
- self.restoreBlob(record.oid, record.tid, record.data,
- name, record.data_txn, trans)
- else:
- self.restore(record.oid, record.tid, record.data,
- '', record.data_txn, trans)
+ copyTransactionsFromTo(other, self)
- self.tpc_vote(trans)
- self.tpc_finish(trans)
-
def loadBlob(self, oid, serial):
"""Return the filename where the blob file can be found.
"""
@@ -943,3 +938,27 @@
pass
return False
+
+def copyTransactionsFromTo(source, destination):
+ for trans in source.iterator():
+ destination.tpc_begin(trans, trans.tid, trans.status)
+ for record in trans:
+ blobfilename = None
+ if is_blob_record(record.data):
+ try:
+ blobfilename = source.loadBlob(record.oid, record.tid)
+ except POSKeyError:
+ pass
+ if blobfilename is not None:
+ fd, name = tempfile.mkstemp(
+ suffix='.tmp', dir=destination.fshelper.temp_dir)
+ os.close(fd)
+ utils.cp(open(blobfilename, 'rb'), open(name, 'wb'))
+ destination.restoreBlob(record.oid, record.tid, record.data,
+ name, record.data_txn, trans)
+ else:
+ destination.restore(record.oid, record.tid, record.data,
+ '', record.data_txn, trans)
+
+ destination.tpc_vote(trans)
+ destination.tpc_finish(trans)
Modified: ZODB/trunk/src/ZODB/interfaces.py
===================================================================
--- ZODB/trunk/src/ZODB/interfaces.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/interfaces.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -287,24 +287,33 @@
"""
-class IStorageDB(Interface):
- """Database interface exposed to storages
+class IStorageWrapper(Interface):
+ """Storage wrapper interface
- This interface provides 2 facilities:
+ This interface provides 3 facilities:
- Out-of-band invalidation support
- A storage can notify it's database of object invalidations that
+ A storage can notify it's wrapper of object invalidations that
don't occur due to direct operations on the storage. Currently
this is only used by ZEO client storages to pass invalidation
messages sent from a server.
- - Record-reference extraction.
+ - Record-reference extraction
The references method can be used to extract referenced object
IDs from a database record. This can be used by storages to
- provide more advanced garbage collection.
+ provide more advanced garbage collection. A wrapper storage
+ that transforms data will provide a references method that
+ untransforms data passed to it and then pass the data to the
+ layer above it.
+ - Record transformation
+
+ A storage wrapper may transform data, for example for
+ compression or encryption. Methods are provided to transform or
+ untransform data.
+
This interface may be implemented by storage adapters or other
intermediaries. For example, a storage adapter that provides
encryption and/or compresssion will apply record transformations
@@ -337,7 +346,17 @@
be created and returned.
"""
+ def transform_record_data(data):
+ """Return transformed data
+ """
+ def untransform_record_data(data):
+ """Return untransformed data
+ """
+
+IStorageDB = IStorageWrapper # for backward compatibility
+
+
class IDatabase(IStorageDB):
"""ZODB DB.
"""
@@ -595,12 +614,18 @@
revisions.
"""
- def registerDB(db):
- """Register an IStorageDB.
+ def registerDB(wrapper):
+ """Register a storage wrapper IStorageWrapper.
+ The passed object is a wrapper object that provides an upcall
+ interface to support composition.
+
Note that, for historical reasons, an implementation may
require a second argument, however, if required, the None will
be passed as the second argument.
+
+ Also, for historical reasons, this is called registerDB rather
+ than register_wrapper.
"""
def sortKey():
Modified: ZODB/trunk/src/ZODB/tests/BasicStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/BasicStorage.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/tests/BasicStorage.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -49,7 +49,7 @@
self.assertRaises(
POSException.StorageTransactionError,
self._storage.store,
- ZERO, 1, 2, '', transaction.Transaction())
+ ZERO, 1, '2', '', transaction.Transaction())
self._storage.tpc_abort(t)
def checkSerialIsNoneForInitialRevision(self):
Copied: ZODB/trunk/src/ZODB/tests/component.xml (from rev 112456, ZODB/branches/jim-transform-wrapping/src/ZODB/tests/component.xml)
===================================================================
--- ZODB/trunk/src/ZODB/tests/component.xml (rev 0)
+++ ZODB/trunk/src/ZODB/tests/component.xml 2010-05-18 15:22:13 UTC (rev 112462)
@@ -0,0 +1,16 @@
+<component>
+ <sectiontype
+ name="hexstorage"
+ datatype="ZODB.tests.hexstorage.ZConfigHex"
+ implements="ZODB.storage"
+ >
+ <section type="ZODB.storage" name="*" attribute="base" required="yes" />
+ </sectiontype>
+ <sectiontype
+ name="serverhexstorage"
+ datatype="ZODB.tests.hexstorage.ZConfigServerHex"
+ implements="ZODB.storage"
+ >
+ <section type="ZODB.storage" name="*" attribute="base" required="yes" />
+ </sectiontype>
+</component>
Copied: ZODB/trunk/src/ZODB/tests/hexstorage.py (from rev 112456, ZODB/branches/jim-transform-wrapping/src/ZODB/tests/hexstorage.py)
===================================================================
--- ZODB/trunk/src/ZODB/tests/hexstorage.py (rev 0)
+++ ZODB/trunk/src/ZODB/tests/hexstorage.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -0,0 +1,161 @@
+##############################################################################
+#
+# Copyright (c) 2010 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import ZODB.blob
+import ZODB.interfaces
+import zope.interface
+
+class HexStorage(object):
+
+ zope.interface.implements(ZODB.interfaces.IStorageWrapper)
+
+ copied_methods = (
+ 'close', 'getName', 'getSize', 'history', 'isReadOnly',
+ 'lastTransaction', 'new_oid', 'sortKey',
+ 'tpc_abort', 'tpc_begin', 'tpc_finish', 'tpc_vote',
+ 'loadBlob', 'openCommittedBlobFile', 'temporaryDirectory',
+ 'supportsUndo', 'undo', 'undoLog', 'undoInfo',
+ )
+
+ def __init__(self, base):
+ self.base = base
+ base.registerDB(self)
+
+ for name in self.copied_methods:
+ v = getattr(base, name, None)
+ if v is not None:
+ setattr(self, name, v)
+
+ zope.interface.directlyProvides(self, zope.interface.providedBy(base))
+
+ def __getattr__(self, name):
+ return getattr(self.base, name)
+
+ def __len__(self):
+ return len(self.base)
+
+ def load(self, oid, version=''):
+ data, serial = self.base.load(oid, version)
+ return data[2:].decode('hex'), serial
+
+ def loadBefore(self, oid, tid):
+ r = self.base.loadBefore(oid, tid)
+ if r is not None:
+ data, serial, after = r
+ return data[2:].decode('hex'), serial, after
+ else:
+ return r
+
+ def loadSerial(self, oid, serial):
+ return self.base.loadSerial(oid, serial)[2:].decode('hex')
+
+ def pack(self, pack_time, referencesf, gc=True):
+ def refs(p, oids=None):
+ return referencesf(p[2:].decode('hex'), oids)
+ return self.base.pack(pack_time, refs, gc)
+
+ def registerDB(self, db):
+ self.db = db
+ self._db_transform = db.transform_record_data
+ self._db_untransform = db.untransform_record_data
+
+ _db_transform = _db_untransform = lambda self, data: data
+
+ def store(self, oid, serial, data, version, transaction):
+ return self.base.store(
+ oid, serial, '.h'+data.encode('hex'), version, transaction)
+
+ def restore(self, oid, serial, data, version, prev_txn, transaction):
+ return self.base.restore(
+ oid, serial, data and ('.h'+data.encode('hex')), version, prev_txn,
+ transaction)
+
+ def iterator(self, start=None, stop=None):
+ for t in self.base.iterator(start, stop):
+ yield Transaction(self, t)
+
+ def storeBlob(self, oid, oldserial, data, blobfilename, version,
+ transaction):
+ return self.base.storeBlob(oid, oldserial, '.h'+data.encode('hex'),
+ blobfilename, version, transaction)
+
+ def restoreBlob(self, oid, serial, data, blobfilename, prev_txn,
+ transaction):
+ return self.base.restoreBlob(oid, serial,
+ data and ('.h'+data.encode('hex')),
+ blobfilename, prev_txn, transaction)
+
+ def invalidateCache(self):
+ return self.db.invalidateCache()
+
+ def invalidate(self, transaction_id, oids, version=''):
+ return self.db.invalidate(transaction_id, oids, version)
+
+ def references(self, record, oids=None):
+ return self.db.references(record[2:].decode('hex'), oids)
+
+ def transform_record_data(self, data):
+ return '.h'+self._db_transform(data).encode('hex')
+
+ def untransform_record_data(self, data):
+ return self._db_untransform(data[2:].decode('hex'))
+
+ def record_iternext(self, next=None):
+ oid, tid, data, next = self.base.record_iternext(next)
+ return oid, tid, data[2:].decode('hex'), next
+
+ def copyTransactionsFrom(self, other):
+ ZODB.blob.copyTransactionsFromTo(other, self)
+
+class ServerHexStorage(HexStorage):
+ """Use on ZEO storage server when Hex is used on client
+
+ Don't do conversion as part of load/store, but provide
+ pickle decoding.
+ """
+
+ copied_methods = HexStorage.copied_methods + (
+ 'load', 'loadBefore', 'loadSerial', 'store', 'restore',
+ 'iterator', 'storeBlob', 'restoreBlob', 'record_iternext',
+ )
+
+class Transaction(object):
+
+ def __init__(self, store, trans):
+ self.__store = store
+ self.__trans = trans
+
+ def __iter__(self):
+ for r in self.__trans:
+ if r.data:
+ r.data = self.__store.untransform_record_data(r.data)
+ yield r
+
+ def __getattr__(self, name):
+ return getattr(self.__trans, name)
+
+class ZConfigHex:
+
+ _factory = HexStorage
+
+ def __init__(self, config):
+ self.config = config
+ self.name = config.getSectionName()
+
+ def open(self):
+ base = self.config.base.open()
+ return self._factory(base)
+
+class ZConfigServerHex(ZConfigHex):
+
+ _factory = ServerHexStorage
Modified: ZODB/trunk/src/ZODB/tests/testDemoStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testDemoStorage.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/tests/testDemoStorage.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -16,6 +16,7 @@
import transaction
from ZODB.DB import DB
from zope.testing import doctest
+import ZODB.tests.hexstorage
import ZODB.tests.util
import ZODB.utils
import ZODB.DemoStorage
@@ -78,6 +79,13 @@
checkUndoZombie = checkLoadBeforeUndo
+class DemoStorageHexTests(DemoStorageTests):
+
+ def setUp(self):
+ StorageTestBase.StorageTestBase.setUp(self)
+ self._storage = ZODB.tests.hexstorage.HexStorage(
+ ZODB.DemoStorage.DemoStorage())
+
class DemoStorageWrappedBase(DemoStorageTests):
def setUp(self):
@@ -111,8 +119,13 @@
from ZODB.FileStorage import FileStorage
return FileStorage('FileStorageTests.fs')
+class DemoStorageWrappedAroundHexMappingStorage(DemoStorageWrappedBase):
+ def _makeBaseStorage(self):
+ from ZODB.MappingStorage import MappingStorage
+ return ZODB.tests.hexstorage.HexStorage(MappingStorage())
+
def setUp(test):
random.seed(0)
ZODB.tests.util.setUp(test)
@@ -232,8 +245,11 @@
),
))
suite.addTest(unittest.makeSuite(DemoStorageTests, 'check'))
+ suite.addTest(unittest.makeSuite(DemoStorageHexTests, 'check'))
suite.addTest(unittest.makeSuite(DemoStorageWrappedAroundFileStorage,
'check'))
suite.addTest(unittest.makeSuite(DemoStorageWrappedAroundMappingStorage,
'check'))
+ suite.addTest(unittest.makeSuite(DemoStorageWrappedAroundHexMappingStorage,
+ 'check'))
return suite
Modified: ZODB/trunk/src/ZODB/tests/testFileStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testFileStorage.py 2010-05-18 15:22:11 UTC (rev 112461)
+++ ZODB/trunk/src/ZODB/tests/testFileStorage.py 2010-05-18 15:22:13 UTC (rev 112462)
@@ -15,6 +15,7 @@
import os, unittest
import transaction
import ZODB.FileStorage
+import ZODB.tests.hexstorage
import ZODB.tests.testblob
import ZODB.tests.util
import zope.testing.setupstack
@@ -279,14 +280,30 @@
else:
self.assertNotEqual(next_oid, None)
+class FileStorageHexTests(FileStorageTests):
+
+ def open(self, **kwargs):
+ self._storage = ZODB.tests.hexstorage.HexStorage(
+ ZODB.FileStorage.FileStorage('FileStorageTests.fs',**kwargs))
+
+
class FileStorageTestsWithBlobsEnabled(FileStorageTests):
def open(self, **kwargs):
if 'blob_dir' not in kwargs:
kwargs = kwargs.copy()
kwargs['blob_dir'] = 'blobs'
- return FileStorageTests.open(self, **kwargs)
+ FileStorageTests.open(self, **kwargs)
+class FileStorageHexTestsWithBlobsEnabled(FileStorageTests):
+
+ def open(self, **kwargs):
+ if 'blob_dir' not in kwargs:
+ kwargs = kwargs.copy()
+ kwargs['blob_dir'] = 'blobs'
+ FileStorageTests.open(self, **kwargs)
+ self._storage = ZODB.tests.hexstorage.HexStorage(self._storage)
+
class FileStorageRecoveryTest(
StorageTestBase.StorageTestBase,
RecoveryStorage.RecoveryStorage,
@@ -304,7 +321,16 @@
def new_dest(self):
return ZODB.FileStorage.FileStorage('Dest.fs')
+class FileStorageHexRecoveryTest(FileStorageRecoveryTest):
+ def setUp(self):
+ StorageTestBase.StorageTestBase.setUp(self)
+ self._storage = ZODB.tests.hexstorage.HexStorage(
+ ZODB.FileStorage.FileStorage("Source.fs", create=True))
+ self._dst = ZODB.tests.hexstorage.HexStorage(
+ ZODB.FileStorage.FileStorage("Dest.fs", create=True))
+
+
class FileStorageNoRestore(ZODB.FileStorage.FileStorage):
@property
@@ -634,10 +660,14 @@
from zope.testing import doctest
suite = unittest.TestSuite()
- for klass in [FileStorageTests, Corruption.FileStorageCorruptTests,
- FileStorageRecoveryTest, FileStorageNoRestoreRecoveryTest,
- FileStorageTestsWithBlobsEnabled, AnalyzeDotPyTest,
- ]:
+ for klass in [
+ FileStorageTests, FileStorageHexTests,
+ Corruption.FileStorageCorruptTests,
+ FileStorageRecoveryTest, FileStorageHexRecoveryTest,
+ FileStorageNoRestoreRecoveryTest,
+ FileStorageTestsWithBlobsEnabled, FileStorageHexTestsWithBlobsEnabled,
+ AnalyzeDotPyTest,
+ ]:
suite.addTest(unittest.makeSuite(klass, "check"))
suite.addTest(doctest.DocTestSuite(
setUp=zope.testing.setupstack.setUpDirectory,
@@ -649,9 +679,18 @@
test_blob_storage_recovery=True,
test_packing=True,
))
+ suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
+ 'BlobFileHexStorage',
+ lambda name, blob_dir:
+ ZODB.tests.hexstorage.HexStorage(
+ ZODB.FileStorage.FileStorage('%s.fs' % name, blob_dir=blob_dir)),
+ test_blob_storage_recovery=True,
+ test_packing=True,
+ ))
suite.addTest(PackableStorage.IExternalGC_suite(
lambda : ZODB.FileStorage.FileStorage(
'data.fs', blob_dir='blobs', pack_gc=False)))
+ suite.layer = ZODB.tests.util.MininalTestLayer('testFileStorage')
return suite
if __name__=='__main__':
More information about the Zodb-checkins
mailing list