[Zodb-checkins] SVN: ZODB/branches/ctheune-bushy-directory/src/
Merge trunk.
Christian Theune
ct at gocept.com
Wed Aug 6 04:57:47 EDT 2008
Log message for revision 89428:
Merge trunk.
Changed:
U ZODB/branches/ctheune-bushy-directory/src/BTrees/Interfaces.py
U ZODB/branches/ctheune-bushy-directory/src/CHANGES.txt
U ZODB/branches/ctheune-bushy-directory/src/ZODB/blob.py
U ZODB/branches/ctheune-bushy-directory/src/ZODB/tests/blob_layout.txt
U ZODB/branches/ctheune-bushy-directory/src/ZODB/tests/blob_packing.txt
-=-
Modified: ZODB/branches/ctheune-bushy-directory/src/BTrees/Interfaces.py
===================================================================
--- ZODB/branches/ctheune-bushy-directory/src/BTrees/Interfaces.py 2008-08-06 08:37:23 UTC (rev 89427)
+++ ZODB/branches/ctheune-bushy-directory/src/BTrees/Interfaces.py 2008-08-06 08:57:47 UTC (rev 89428)
@@ -140,13 +140,19 @@
Return the default if has_key(key) is false.
"""
+ def __getitem__(key):
+ """Get the value associated with the given key.
+
+ Raise KeyError if has_key(key) is false.
+ """
+
def __setitem__(key, value):
"""Set the value associated with the given key."""
def __delitem__(key):
"""Delete the value associated with the given key.
- Raise KeyError if the key if has_key(key) is false.
+ Raise KeyError if has_key(key) is false.
"""
def values(min=None, max=None, excludemin=False, excludemax=False):
Modified: ZODB/branches/ctheune-bushy-directory/src/CHANGES.txt
===================================================================
--- ZODB/branches/ctheune-bushy-directory/src/CHANGES.txt 2008-08-06 08:37:23 UTC (rev 89427)
+++ ZODB/branches/ctheune-bushy-directory/src/CHANGES.txt 2008-08-06 08:57:47 UTC (rev 89428)
@@ -43,8 +43,10 @@
Bugs Fixed
----------
-- Fix for bug# 220856: Completed implementation of ZEO authentication.
+- Fix for bug #251037: Make packing of blob storages non-blocking.
+- Fix for bug #220856: Completed implementation of ZEO authentication.
+
- Fix for bug #184057: Make initialisation of small ZEO client file cache
sizes not fail.
Modified: ZODB/branches/ctheune-bushy-directory/src/ZODB/blob.py
===================================================================
--- ZODB/branches/ctheune-bushy-directory/src/ZODB/blob.py 2008-08-06 08:37:23 UTC (rev 89427)
+++ ZODB/branches/ctheune-bushy-directory/src/ZODB/blob.py 2008-08-06 08:57:47 UTC (rev 89428)
@@ -530,6 +530,10 @@
LAYOUTS['lawn'] = LawnLayout()
+class BlobStorageError(Exception):
+ """The blob storage encountered an invalid state."""
+
+
class BlobStorage(SpecificationDecoratorBase):
"""A storage to support blobs."""
@@ -537,7 +541,8 @@
# Proxies can't have a __dict__ so specifying __slots__ here allows
# us to have instance attributes explicitly on the proxy.
- __slots__ = ('fshelper', 'dirty_oids', '_BlobStorage__supportsUndo')
+ __slots__ = ('fshelper', 'dirty_oids', '_BlobStorage__supportsUndo',
+ '_blobs_pack_is_in_progress', )
def __new__(self, base_directory, storage, layout='automatic'):
return SpecificationDecoratorBase.__new__(self, storage)
@@ -556,6 +561,7 @@
else:
supportsUndo = supportsUndo()
self.__supportsUndo = supportsUndo
+ self._blobs_pack_is_in_progress = False
@non_overridable
def temporaryDirectory(self):
@@ -660,21 +666,29 @@
@non_overridable
def pack(self, packtime, referencesf):
- """Remove all unused oid/tid combinations."""
- unproxied = getProxiedObject(self)
+ """Remove all unused OID/TID combinations."""
+ self._lock_acquire()
+ try:
+ if self._blobs_pack_is_in_progress:
+ raise BlobStorageError('Already packing')
+ self._blobs_pack_is_in_progress = True
+ finally:
+ self._lock_release()
- # pack the underlying storage, which will allow us to determine
- # which serials are current.
- result = unproxied.pack(packtime, referencesf)
+ try:
+ # Pack the underlying storage, which will allow us to determine
+ # which serials are current.
+ unproxied = getProxiedObject(self)
+ result = unproxied.pack(packtime, referencesf)
- # perform a pack on blob data
- self._lock_acquire()
- try:
+ # Perform a pack on the blob data.
if self.__supportsUndo:
self._packUndoing(packtime, referencesf)
else:
self._packNonUndoing(packtime, referencesf)
finally:
+ self._lock_acquire()
+ self._blobs_pack_is_in_progress = False
self._lock_release()
return result
Modified: ZODB/branches/ctheune-bushy-directory/src/ZODB/tests/blob_layout.txt
===================================================================
--- ZODB/branches/ctheune-bushy-directory/src/ZODB/tests/blob_layout.txt 2008-08-06 08:37:23 UTC (rev 89427)
+++ ZODB/branches/ctheune-bushy-directory/src/ZODB/tests/blob_layout.txt 2008-08-06 08:57:47 UTC (rev 89428)
@@ -207,9 +207,9 @@
>>> bushy = os.path.join(d, 'bushy')
>>> migrate(old, bushy, 'bushy') # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Migrating blob data from `/.../old` (lawn) to `/.../bushy` (bushy)
- OID: 0x1b7f - 2 files
- OID: 0x0a - 2 files
- OID: 0x1b7a - 2 files
+ OID: 0x1b7a - 2 files
+ OID: 0x0a - 2 files
+ OID: 0x1b7f - 2 files
The new directory now contains the same files in different directories, but
with the same sizes and permissions:
@@ -226,6 +226,26 @@
>>> ls(bushy)
040700 4096 /.../bushy
0100644 5 /.../bushy/.layout
+ 040700 4096 /.../bushy/0x7f
+ 040700 4096 /.../bushy/0x7f/0x1b
+ 040700 4096 /.../bushy/0x7f/0x1b/0x00
+ 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00
+ 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00/0x00
+ 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00
+ 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00/0x00
+ 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00/0x00/0x00
+ 0100644 3 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00/0x00/0x00/foo2
+ 0100644 3 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00/0x00/0x00/foo
+ 040700 4096 /.../bushy/0x0a
+ 040700 4096 /.../bushy/0x0a/0x00
+ 040700 4096 /.../bushy/0x0a/0x00/0x00
+ 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00
+ 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00/0x00
+ 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00
+ 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00/0x00
+ 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00/0x00/0x00
+ 0100644 3 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00/0x00/0x00/foo4
+ 0100644 3 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00/0x00/0x00/foo3
040700 4096 /.../bushy/0x7a
040700 4096 /.../bushy/0x7a/0x1b
040700 4096 /.../bushy/0x7a/0x1b/0x00
@@ -234,50 +254,30 @@
040700 4096 /.../bushy/0x7a/0x1b/0x00/0x00/0x00/0x00
040700 4096 /.../bushy/0x7a/0x1b/0x00/0x00/0x00/0x00/0x00
040700 4096 /.../bushy/0x7a/0x1b/0x00/0x00/0x00/0x00/0x00/0x00
+ 0100644 4 /.../bushy/0x7a/0x1b/0x00/0x00/0x00/0x00/0x00/0x00/foo5
0100644 5 /.../bushy/0x7a/0x1b/0x00/0x00/0x00/0x00/0x00/0x00/foo6
- 0100644 4 /.../bushy/0x7a/0x1b/0x00/0x00/0x00/0x00/0x00/0x00/foo5
040700 4096 /.../bushy/tmp
- 040700 4096 /.../bushy/0x0a
- 040700 4096 /.../bushy/0x0a/0x00
- 040700 4096 /.../bushy/0x0a/0x00/0x00
- 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00
- 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00/0x00
- 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00
- 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00/0x00
- 040700 4096 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00/0x00/0x00
- 0100644 3 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00/0x00/0x00/foo4
- 0100644 3 /.../bushy/0x0a/0x00/0x00/0x00/0x00/0x00/0x00/0x00/foo3
- 040700 4096 /.../bushy/0x7f
- 040700 4096 /.../bushy/0x7f/0x1b
- 040700 4096 /.../bushy/0x7f/0x1b/0x00
- 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00
- 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00/0x00
- 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00
- 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00/0x00
- 040700 4096 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00/0x00/0x00
- 0100644 3 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00/0x00/0x00/foo
- 0100644 3 /.../bushy/0x7f/0x1b/0x00/0x00/0x00/0x00/0x00/0x00/foo2
We can also migrate the bushy layout back to the lawn layout:
>>> lawn = os.path.join(d, 'lawn')
>>> migrate(bushy, lawn, 'lawn')
Migrating blob data from `/.../bushy` (bushy) to `/.../lawn` (lawn)
+ OID: 0x1b7f - 2 files
+ OID: 0x0a - 2 files
OID: 0x1b7a - 2 files
- OID: 0x0a - 2 files
- OID: 0x1b7f - 2 files
>>> ls(lawn)
040700 4096 /.../lawn
0100644 4 /.../lawn/.layout
+ 040700 4096 /.../lawn/0x1b7a
+ 0100644 4 /.../lawn/0x1b7a/foo5
+ 0100644 5 /.../lawn/0x1b7a/foo6
+ 040700 4096 /.../lawn/0x0a
+ 0100644 3 /.../lawn/0x0a/foo4
+ 0100644 3 /.../lawn/0x0a/foo3
040700 4096 /.../lawn/0x1b7f
+ 0100644 3 /.../lawn/0x1b7f/foo2
0100644 3 /.../lawn/0x1b7f/foo
- 0100644 3 /.../lawn/0x1b7f/foo2
040700 4096 /.../lawn/tmp
- 040700 4096 /.../lawn/0x0a
- 0100644 3 /.../lawn/0x0a/foo4
- 0100644 3 /.../lawn/0x0a/foo3
- 040700 4096 /.../lawn/0x1b7a
- 0100644 5 /.../lawn/0x1b7a/foo6
- 0100644 4 /.../lawn/0x1b7a/foo5
>>> shutil.rmtree(d)
Modified: ZODB/branches/ctheune-bushy-directory/src/ZODB/tests/blob_packing.txt
===================================================================
--- ZODB/branches/ctheune-bushy-directory/src/ZODB/tests/blob_packing.txt 2008-08-06 08:37:23 UTC (rev 89427)
+++ ZODB/branches/ctheune-bushy-directory/src/ZODB/tests/blob_packing.txt 2008-08-06 08:57:47 UTC (rev 89428)
@@ -240,6 +240,37 @@
>>> os.path.exists(os.path.split(fns[0])[0])
False
+Avoiding parallel packs
+=======================
+
+Blob packing (similar to FileStorage) can only be run once at a time. For
+this, a flag (_blobs_pack_is_in_progress) is set. If the pack method is called
+while this flag is set, it will refuse to perform another pack, until the flag
+is reset:
+
+ >>> blob_storage._blobs_pack_is_in_progress
+ False
+ >>> blob_storage._blobs_pack_is_in_progress = True
+ >>> blob_storage.pack(packtime, referencesf)
+ Traceback (most recent call last):
+ BlobStorageError: Already packing
+ >>> blob_storage._blobs_pack_is_in_progress = False
+ >>> blob_storage.pack(packtime, referencesf)
+
+We can also see, that the flag is set during the pack, by leveraging the
+knowledge that the underlying storage's pack method is also called:
+
+ >>> def dummy_pack(time, ref):
+ ... print "_blobs_pack_is_in_progress =", blob_storage._blobs_pack_is_in_progress
+ ... return base_pack(time, ref)
+ >>> base_pack = base_storage.pack
+ >>> base_storage.pack = dummy_pack
+ >>> blob_storage.pack(packtime, referencesf)
+ _blobs_pack_is_in_progress = True
+ >>> blob_storage._blobs_pack_is_in_progress
+ False
+ >>> base_storage.pack = base_pack
+
Clean up our blob directory:
>>> shutil.rmtree(blob_dir)
More information about the Zodb-checkins
mailing list