[Zodb-checkins]
SVN: ZODB/branches/ctheune-bushy-directory-3.8/src/Z
ported getSize() and TmpStore changes for the 3.8 branch
Christian Theune
ct at gocept.com
Thu Jun 26 15:53:05 EDT 2008
Log message for revision 87812:
ported getSize() and TmpStore changes for the 3.8 branch
Changed:
U ZODB/branches/ctheune-bushy-directory-3.8/src/ZEO/ClientStorage.py
U ZODB/branches/ctheune-bushy-directory-3.8/src/ZEO/tests/testZEO.py
U ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/Connection.py
U ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/blob.py
U ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/tests/blob_transaction.txt
U ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/tests/testblob.py
-=-
Modified: ZODB/branches/ctheune-bushy-directory-3.8/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/ctheune-bushy-directory-3.8/src/ZEO/ClientStorage.py 2008-06-26 19:32:58 UTC (rev 87811)
+++ ZODB/branches/ctheune-bushy-directory-3.8/src/ZEO/ClientStorage.py 2008-06-26 19:53:05 UTC (rev 87812)
@@ -1036,7 +1036,7 @@
pass
def temporaryDirectory(self):
- return self.blob_dir
+ return self.fshelper.temp_dir
def tpc_vote(self, txn):
"""Storage API: vote on a transaction."""
Modified: ZODB/branches/ctheune-bushy-directory-3.8/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/branches/ctheune-bushy-directory-3.8/src/ZEO/tests/testZEO.py 2008-06-26 19:32:58 UTC (rev 87811)
+++ ZODB/branches/ctheune-bushy-directory-3.8/src/ZEO/tests/testZEO.py 2008-06-26 19:53:05 UTC (rev 87812)
@@ -564,7 +564,7 @@
self.assert_((os.stat(filename).st_mode & stat.S_IREAD))
def checkTemporaryDirectory(self):
- self.assertEquals(self.blob_cache_dir,
+ self.assertEquals(os.path.join(self.blob_cache_dir, 'tmp'),
self._storage.temporaryDirectory())
def checkTransactionBufferCleanup(self):
Modified: ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/Connection.py
===================================================================
--- ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/Connection.py 2008-06-26 19:32:58 UTC (rev 87811)
+++ ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/Connection.py 2008-06-26 19:53:05 UTC (rev 87812)
@@ -1244,7 +1244,7 @@
transaction):
serial = self.store(oid, serial, data, version, transaction)
- targetpath = self._getBlobPath(oid)
+ targetpath = self._getBlobPath()
if not os.path.exists(targetpath):
os.makedirs(targetpath, 0700)
@@ -1263,14 +1263,12 @@
return self._storage.loadBlob(oid, serial)
return filename
- def _getBlobPath(self, oid):
- return os.path.join(self.temporaryDirectory(),
- utils.oid_repr(oid)
- )
+ def _getBlobPath(self):
+ return os.path.join(self.temporaryDirectory(), 'savepoints')
def _getCleanFilename(self, oid, tid):
- return os.path.join(self._getBlobPath(oid),
- "%s%s" % (utils.tid_repr(tid), SAVEPOINT_SUFFIX,)
+ return os.path.join(self._getBlobPath(),
+ "%s-%s%s" % (utils.oid_repr(oid), utils.tid_repr(tid), SAVEPOINT_SUFFIX,)
)
def temporaryDirectory(self):
Modified: ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/blob.py
===================================================================
--- ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/blob.py 2008-06-26 19:32:58 UTC (rev 87811)
+++ ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/blob.py 2008-06-26 19:53:05 UTC (rev 87812)
@@ -683,17 +683,10 @@
@non_overridable
def getSize(self):
"""Return the size of the database in bytes."""
- orig_size = getProxiedObject(self).getSize()
- blob_size = 0
- for oid, path in self.fshelper.listOIDs():
- for serial in os.listdir(path):
- if not serial.endswith(BLOB_SUFFIX):
- continue
- file_path = os.path.join(path, serial)
- blob_size += os.stat(file_path).st_size
+ # XXX The old way of computing is way to resource hungry. We need to
+ # do some kind of estimation instead.
+ return getProxiedObject(self).getSize()
- return orig_size + blob_size
-
@non_overridable
def undo(self, serial_id, transaction):
undo_serial, keys = getProxiedObject(self).undo(serial_id, transaction)
Modified: ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/tests/blob_transaction.txt
===================================================================
--- ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/tests/blob_transaction.txt 2008-06-26 19:32:58 UTC (rev 87811)
+++ ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/tests/blob_transaction.txt 2008-06-26 19:53:05 UTC (rev 87812)
@@ -191,15 +191,15 @@
>>> root4['blob1'].open('r').read()
'this is blob 1woot!this is from connection 3'
-BlobStorages implementation of getSize() includes the blob data and adds it to
-the underlying storages result of getSize(). (We need to ensure the last
+BlobStorages implementation of getSize() does not include the blob data and
+only returns what the underlying storages do. (We need to ensure the last
number to be an int, otherwise it will be a long on 32-bit platforms and an
int on 64-bit)::
>>> underlying_size = base_storage.getSize()
>>> blob_size = blob_storage.getSize()
>>> int(blob_size - underlying_size)
- 91
+ 0
You can't commit a transaction while blob files are open:
@@ -243,22 +243,53 @@
>>> root5['blob'].open("r").read()
"I'm a happy blob. And I'm singing."
>>> savepoint = transaction.savepoint(optimistic=True)
+
>>> root5['blob'].open("r").read()
"I'm a happy blob. And I'm singing."
+
+Savepoints store the blobs in the `savepoints` directory in the temporary
+directory of the blob storage:
+
+ >>> os.listdir(os.path.join(blob_dir, 'tmp', 'savepoints'))
+ ['0x03-0x....spb']
>>> transaction.commit()
+After committing the transaction, the temporary savepoint files are moved to
+the committed location again:
+
+ >>> os.listdir(os.path.join(blob_dir, 'tmp', 'savepoints'))
+ []
+
We support non-optimistic savepoints too:
>>> root5['blob'].open("a").write(" And I'm dancing.")
>>> root5['blob'].open("r").read()
"I'm a happy blob. And I'm singing. And I'm dancing."
>>> savepoint = transaction.savepoint()
+
+Again, the savepoint creates a new file for the blob state in the savepoints
+directory:
+
+ >>> os.listdir(os.path.join(blob_dir, 'tmp', 'savepoints'))
+ ['0x03-0x....spb']
+
>>> root5['blob'].open("w").write(" And the weather is beautiful.")
>>> savepoint.rollback()
+
+XXX Currently, savepoint state of blobs remains after a rollback:
+
+ >>> os.listdir(os.path.join(blob_dir, 'tmp', 'savepoints'))
+ ['0x03-0x....spb']
+
>>> root5['blob'].open("r").read()
"I'm a happy blob. And I'm singing. And I'm dancing."
>>> transaction.abort()
+XXX Currently, savepoint state of blobs remains even after an abort:
+
+ >>> os.listdir(os.path.join(blob_dir, 'tmp', 'savepoints'))
+ ['0x03-0x....spb']
+
Reading Blobs outside of a transaction
--------------------------------------
@@ -341,21 +372,7 @@
Note: This is a counter measure against regression of bug #126007.
-getSize with garbage in the directory structure
------------------------------------------------
-`getSize` iterates over the existing blob files in the blob directory and adds
-up their size. The blob directory sometimes contains temporary files that the
-getSize function needs to ignore:
-
- >>> garbage_file = os.path.join(blob_dir, 'garbage')
- >>> open(garbage_file, 'w').write('garbage')
- >>> int(blob_storage.getSize())
- 881
-
-
-Note: This is a counter measer against regression of bug #12991.
-
Teardown
--------
Modified: ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/tests/testblob.py
===================================================================
--- ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/tests/testblob.py 2008-06-26 19:32:58 UTC (rev 87811)
+++ ZODB/branches/ctheune-bushy-directory-3.8/src/ZODB/tests/testblob.py 2008-06-26 19:53:05 UTC (rev 87812)
@@ -503,6 +503,7 @@
"blob_basic.txt", "blob_connection.txt", "blob_transaction.txt",
"blob_packing.txt", "blob_importexport.txt", "blob_consume.txt",
"blob_tempdir.txt",
+ optionflags=doctest.ELLIPSIS,
setUp=ZODB.tests.util.setUp,
tearDown=ZODB.tests.util.tearDown,
))
More information about the Zodb-checkins
mailing list