[Zodb-checkins] SVN: ZODB/trunk/src/ Fixed a couple of blob storage issues:
Shane Hathaway
shane at hathawaymix.org
Fri Jul 10 19:41:51 EDT 2009
Log message for revision 101802:
Fixed a couple of blob storage issues:
- The "lawn" layout was being selected by default if the root of
the blob directory happened to contain a hidden file or directory
such as ".svn". Now hidden files and directories are ignored
when choosing the default layout.
- BlobStorage was not compatible with MVCC storages because the
wrappers were being removed by each database connection. There
was also a problem with subtransactions. Fixed.
Changed:
U ZODB/trunk/src/CHANGES.txt
U ZODB/trunk/src/ZODB/Connection.py
U ZODB/trunk/src/ZODB/blob.py
U ZODB/trunk/src/ZODB/tests/blob_layout.txt
U ZODB/trunk/src/ZODB/tests/blob_packing.txt
U ZODB/trunk/src/ZODB/tests/blob_transaction.txt
U ZODB/trunk/src/ZODB/tests/testMVCCMappingStorage.py
U ZODB/trunk/src/ZODB/tests/testblob.py
-=-
Modified: ZODB/trunk/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt 2009-07-10 22:20:55 UTC (rev 101801)
+++ ZODB/trunk/src/CHANGES.txt 2009-07-10 23:41:51 UTC (rev 101802)
@@ -22,6 +22,14 @@
- zeopack was less flexible than it was before. -h should default to
local host.
+- The "lawn" layout was being selected by default if the root of
+ the blob directory happened to contain a hidden file or directory
+ such as ".svn". Now hidden files and directories are ignored
+ when choosing the default layout.
+
+- BlobStorage was not compatible with MVCC storages because the
+ wrappers were being removed by each database connection. Fixed.
+
3.9.0b2 (2009-06-11)
====================
Modified: ZODB/trunk/src/ZODB/Connection.py
===================================================================
--- ZODB/trunk/src/ZODB/Connection.py 2009-07-10 22:20:55 UTC (rev 101801)
+++ ZODB/trunk/src/ZODB/Connection.py 2009-07-10 23:41:51 UTC (rev 101802)
@@ -334,10 +334,6 @@
def invalidate(self, tid, oids):
"""Notify the Connection that transaction 'tid' invalidated oids."""
- if self._mvcc_storage:
- # Inter-connection invalidation is not needed when the
- # storage provides MVCC.
- return
if self.before is not None:
# this is an historical connection. Invalidations are irrelevant.
return
@@ -771,6 +767,10 @@
"""Indicate confirmation that the transaction is done."""
def callback(tid):
+ if self._mvcc_storage:
+ # Inter-connection invalidation is not needed when the
+ # storage provides MVCC.
+ return
d = dict.fromkeys(self._modified)
self._db.invalidate(tid, d, self)
# It's important that the storage calls the passed function
Modified: ZODB/trunk/src/ZODB/blob.py
===================================================================
--- ZODB/trunk/src/ZODB/blob.py 2009-07-10 22:20:55 UTC (rev 101801)
+++ ZODB/trunk/src/ZODB/blob.py 2009-07-10 23:41:51 UTC (rev 101802)
@@ -502,23 +502,30 @@
# A heuristic to look at a path and determine which directory layout to
# use.
layout_marker = os.path.join(path, LAYOUT_MARKER)
- if not os.path.exists(path):
+ if os.path.exists(layout_marker):
+ layout = open(layout_marker, 'rb').read()
+ layout = layout.strip()
+ log('Blob directory `%s` has layout marker set. '
+ 'Selected `%s` layout. ' % (path, layout), level=logging.DEBUG)
+ elif not os.path.exists(path):
log('Blob directory %s does not exist. '
'Selected `bushy` layout. ' % path)
layout = 'bushy'
- elif len(os.listdir(path)) == 0:
- log('Blob directory `%s` is unused and has no layout marker set. '
- 'Selected `bushy` layout. ' % path)
- layout = 'bushy'
- elif LAYOUT_MARKER not in os.listdir(path):
- log('Blob directory `%s` is used but has no layout marker set. '
- 'Selected `lawn` layout. ' % path)
- layout = 'lawn'
else:
- layout = open(layout_marker, 'rb').read()
- layout = layout.strip()
- log('Blob directory `%s` has layout marker set. '
- 'Selected `%s` layout. ' % (path, layout), level=logging.DEBUG)
+ # look for a non-hidden file in the directory
+ has_files = False
+ for name in os.listdir(path):
+ if not name.startswith('.'):
+ has_files = True
+ break
+ if not has_files:
+ log('Blob directory `%s` is unused and has no layout marker set. '
+ 'Selected `bushy` layout. ' % path)
+ layout = 'bushy'
+ else:
+ log('Blob directory `%s` is used but has no layout marker set. '
+ 'Selected `lawn` layout. ' % path)
+ layout = 'lawn'
return layout
@@ -861,7 +868,19 @@
self._lock_release()
return undo_serial, keys
+ @non_overridable
+ def new_instance(self):
+ """Implementation of IMVCCStorage.new_instance.
+ This method causes all storage instances to be wrapped with
+ a blob storage wrapper.
+ """
+ base_dir = self.fshelper.base_dir
+ s = getProxiedObject(self).new_instance()
+ res = BlobStorage(base_dir, s)
+ return res
+
+
for name, v in BlobStorageMixin.__dict__.items():
if isinstance(v, type(BlobStorageMixin.__dict__['storeBlob'])):
assert name not in BlobStorage.__dict__
Modified: ZODB/trunk/src/ZODB/tests/blob_layout.txt
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_layout.txt 2009-07-10 22:20:55 UTC (rev 101801)
+++ ZODB/trunk/src/ZODB/tests/blob_layout.txt 2009-07-10 23:41:51 UTC (rev 101802)
@@ -106,9 +106,9 @@
'lawn'
>>> shutil.rmtree('blobs')
-4. If the directory does not contain a marker but other files, we assume that
-it was created with an earlier version of the blob implementation and uses our
-`lawn` layout:
+4. If the directory does not contain a marker but other files that are
+not hidden, we assume that it was created with an earlier version of
+the blob implementation and uses our `lawn` layout:
>>> os.mkdir('blobs')
>>> open(os.path.join('blobs', '0x0101'), 'wb').write('foo')
@@ -116,7 +116,15 @@
'lawn'
>>> shutil.rmtree('blobs')
+5. If the directory contains only hidden files, use the bushy layout:
+>>> os.mkdir('blobs')
+>>> open(os.path.join('blobs', '.svn'), 'wb').write('blah')
+>>> auto_layout_select('blobs')
+'bushy'
+>>> shutil.rmtree('blobs')
+
+
Directory layout markers
========================
Modified: ZODB/trunk/src/ZODB/tests/blob_packing.txt
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_packing.txt 2009-07-10 22:20:55 UTC (rev 101801)
+++ ZODB/trunk/src/ZODB/tests/blob_packing.txt 2009-07-10 23:41:51 UTC (rev 101802)
@@ -37,7 +37,7 @@
>>> blob.open('w').write('this is blob data 0')
>>> root['blob'] = blob
>>> transaction.commit()
- >>> tids.append(blob_storage._tid)
+ >>> tids.append(blob._p_serial)
>>> nothing = transaction.begin()
>>> times.append(new_time())
Modified: ZODB/trunk/src/ZODB/tests/blob_transaction.txt
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_transaction.txt 2009-07-10 22:20:55 UTC (rev 101801)
+++ ZODB/trunk/src/ZODB/tests/blob_transaction.txt 2009-07-10 23:41:51 UTC (rev 101802)
@@ -174,7 +174,7 @@
>>> tm2.commit()
Traceback (most recent call last):
...
- ConflictError: database conflict error (oid 0x01, class ZODB.blob.Blob)
+ ConflictError: database conflict error (oid 0x01, class ZODB.blob.Blob...)
After the conflict, the winning transaction's result is visible on both
connections::
Modified: ZODB/trunk/src/ZODB/tests/testMVCCMappingStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testMVCCMappingStorage.py 2009-07-10 22:20:55 UTC (rev 101801)
+++ ZODB/trunk/src/ZODB/tests/testMVCCMappingStorage.py 2009-07-10 23:41:51 UTC (rev 101802)
@@ -18,8 +18,9 @@
import transaction
from ZODB.DB import DB
from ZODB.tests.MVCCMappingStorage import MVCCMappingStorage
+import ZODB.blob
+import ZODB.tests.testblob
-
from ZODB.tests import (
BasicStorage,
HistoryStorage,
@@ -167,9 +168,20 @@
self._storage.tpc_begin(t)
self.assertEqual(self._storage._tid, 'zzzzzzzz')
+def create_blob_storage(name, blob_dir):
+ s = MVCCMappingStorage(name)
+ return ZODB.blob.BlobStorage(blob_dir, s)
def test_suite():
suite = unittest.makeSuite(MVCCMappingStorageTests, 'check')
+ # Note: test_packing doesn't work because even though MVCCMappingStorage
+ # retains history, it does not provide undo methods, so the
+ # BlobStorage wrapper calls _packNonUndoing instead of _packUndoing,
+ # causing blobs to get deleted even though object states are retained.
+ suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
+ 'MVCCMapping', create_blob_storage,
+ test_undo=False,
+ ))
return suite
if __name__ == "__main__":
Modified: ZODB/trunk/src/ZODB/tests/testblob.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testblob.py 2009-07-10 22:20:55 UTC (rev 101801)
+++ ZODB/trunk/src/ZODB/tests/testblob.py 2009-07-10 23:41:51 UTC (rev 101802)
@@ -476,7 +476,7 @@
>>> import transaction
>>> transaction.commit()
>>> blob_oid = root['blob']._p_oid
- >>> tid = blob_storage.lastTransaction()
+ >>> tid = connection._storage.lastTransaction()
Now we open a database with a TmpStore in front:
@@ -556,6 +556,7 @@
def storage_reusable_suite(prefix, factory,
test_blob_storage_recovery=False,
test_packing=False,
+ test_undo=True,
):
"""Return a test suite for a generic IBlobStorage.
@@ -605,7 +606,8 @@
if test_blob_storage_recovery:
add_test_based_on_test_class(RecoveryBlobStorage)
- add_test_based_on_test_class(BlobUndoTests)
+ if test_undo:
+ add_test_based_on_test_class(BlobUndoTests)
suite.layer = ZODB.tests.util.MininalTestLayer(prefix+'BlobTests')
More information about the Zodb-checkins
mailing list