[Zodb-checkins] SVN: ZODB/branches/jim-dev/src/ZODB/ checkpoint
Jim Fulton
jim at zope.com
Thu Dec 11 18:18:23 EST 2008
Log message for revision 93921:
checkpoint
Changed:
U ZODB/branches/jim-dev/src/ZODB/FileStorage/FileStorage.py
U ZODB/branches/jim-dev/src/ZODB/blob.py
U ZODB/branches/jim-dev/src/ZODB/tests/testFileStorage.py
-=-
Modified: ZODB/branches/jim-dev/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/branches/jim-dev/src/ZODB/FileStorage/FileStorage.py 2008-12-11 23:16:10 UTC (rev 93920)
+++ ZODB/branches/jim-dev/src/ZODB/FileStorage/FileStorage.py 2008-12-11 23:18:23 UTC (rev 93921)
@@ -16,34 +16,36 @@
$Revision: 1.16 $
"""
+from cPickle import Pickler, Unpickler, loads
+from persistent.TimeStamp import TimeStamp
+from struct import pack, unpack
+from types import StringType
+from zc.lockfile import LockFile
+from ZODB.FileStorage.format import CorruptedDataError
+from ZODB.FileStorage.format import FileStorageFormatter, DataHeader
+from ZODB.FileStorage.format import TRANS_HDR, TRANS_HDR_LEN
+from ZODB.FileStorage.format import TxnHeader, DATA_HDR, DATA_HDR_LEN
+from ZODB.FileStorage.fspack import FileStoragePacker
+from ZODB.fsIndex import fsIndex
+from ZODB import BaseStorage, ConflictResolution, POSException
+from ZODB.loglevels import BLATHER
+from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
+from ZODB.utils import p64, u64, z64
+
import base64
-from cPickle import Pickler, Unpickler, loads
import errno
+import logging
import os
import sys
import time
-import logging
-from types import StringType
-from struct import pack, unpack
+import ZODB.blob
+import ZODB.interfaces
+import zope.interface
+import ZODB.utils
# Not all platforms have fsync
fsync = getattr(os, "fsync", None)
-import zope.interface
-import ZODB.interfaces
-from ZODB import BaseStorage, ConflictResolution, POSException
-from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
-from persistent.TimeStamp import TimeStamp
-from zc.lockfile import LockFile
-from ZODB.utils import p64, u64, cp, z64
-from ZODB.FileStorage.fspack import FileStoragePacker
-from ZODB.FileStorage.format import FileStorageFormatter, DataHeader
-from ZODB.FileStorage.format import TxnHeader, DATA_HDR, DATA_HDR_LEN
-from ZODB.FileStorage.format import TRANS_HDR, TRANS_HDR_LEN
-from ZODB.FileStorage.format import CorruptedDataError
-from ZODB.loglevels import BLATHER
-from ZODB.fsIndex import fsIndex
-
packed_version = "FS21"
logger = logging.getLogger('ZODB.FileStorage')
@@ -86,9 +88,12 @@
def __init__(self, afile):
self._file = afile
-class FileStorage(BaseStorage.BaseStorage,
- ConflictResolution.ConflictResolvingStorage,
- FileStorageFormatter):
+class FileStorage(
+ FileStorageFormatter,
+ ZODB.blob.BlobStorageMixin,
+ ConflictResolution.ConflictResolvingStorage,
+ BaseStorage.BaseStorage,
+ ):
zope.interface.implements(
ZODB.interfaces.IStorage,
@@ -102,7 +107,7 @@
_pack_is_in_progress = False
def __init__(self, file_name, create=False, read_only=False, stop=None,
- quota=None, pack_gc=True, packer=None):
+ quota=None, pack_gc=True, packer=None, blob_dir=None):
if read_only:
self._is_read_only = True
@@ -198,6 +203,15 @@
self._quota = quota
+ self.blob_dir = blob_dir
+ if blob_dir:
+ self._blob_init(blob_dir)
+ zope.interface.alsoProvides(self,
+ ZODB.interfaces.IBlobStorageRestoreable)
+ else:
+ self._blob_init_no_blobs()
+
+
def _initIndex(self, index, tindex):
self._index=index
self._tindex=tindex
@@ -654,7 +668,7 @@
h.descr = descr
h.ext = ext
self._file.write(h.asString())
- cp(self._tfile, self._file, dlen)
+ ZODB.utils.cp(self._tfile, self._file, dlen)
self._file.write(p64(tl))
self._file.flush()
except:
@@ -695,11 +709,13 @@
self._pos = self._nextpos
self._index.update(self._tindex)
self._ltid = tid
+ self._blob_tpc_finish()
def _abort(self):
if self._nextpos:
self._file.truncate(self._pos)
self._nextpos=0
+ self._blob_tpc_abort()
def _undoDataInfo(self, oid, pos, tpos):
"""Return the tid, data pointer, and data for the oid record at pos
@@ -920,6 +936,18 @@
# Don't fail right away. We may be redeemed later!
failures[h.oid] = v
else:
+
+ if self.blob_dir and not p and prev:
+ up, userial = self._loadBackTxn(h.oid, prev)
+ if ZODB.blob.is_blob_record(up):
+ # We're undoing a blob modification operation.
+ # We have to copy the blob data
+ tmp = ZODB.utils.mktemp(dir=self.fshelper.temp_dir)
+ ZODB.utils.cp(
+ self.openCommittedBlobFile(h.oid, userial),
+ open(tmp, 'wb'))
+ self._blob_storeblob(h.oid, self._tid, tmp)
+
new = DataHeader(h.oid, self._tid, ipos, otloc, 0, len(p))
# TODO: This seek shouldn't be necessary, but some other
@@ -1469,7 +1497,7 @@
name, oname)
o = open(oname,'wb')
file.seek(pos)
- cp(file, o, file_size-pos)
+ ZODB.utils.cp(file, o, file_size-pos)
o.close()
break
except:
Modified: ZODB/branches/jim-dev/src/ZODB/blob.py
===================================================================
--- ZODB/branches/jim-dev/src/ZODB/blob.py 2008-12-11 23:16:10 UTC (rev 93920)
+++ ZODB/branches/jim-dev/src/ZODB/blob.py 2008-12-11 23:18:23 UTC (rev 93921)
@@ -483,7 +483,15 @@
continue
yield oid, path
+class NoBlobsFileSystemHelper:
+ @property
+ def temp_dir(self):
+ raise TypeError("Blobs are not supported")
+
+ getPathForOID = getBlobFilenamem = temp_dir
+
+
class BlobStorageError(Exception):
"""The blob storage encountered an invalid state."""
@@ -577,50 +585,34 @@
zope.interface.implements(ZODB.interfaces.IBlobStorage)
- def __init__(self, blob_dir, layout='automatic'):
+ @non_overridable
+ def _blob_init(self, blob_dir, layout='automatic'):
# XXX Log warning if storage is ClientStorage
self.fshelper = FilesystemHelper(blob_dir, layout)
self.fshelper.create()
self.fshelper.checkSecure()
self.dirty_oids = []
- def temporaryDirectory(self):
- return self.fshelper.temp_dir
+ def _blob_init_no_blobs(self):
+ self.fshelper = NoBlobsFileSystemHelper()
+ self.dirty_oids = []
@non_overridable
- def _storeblob(self, oid, serial, blobfilename):
- self._lock_acquire()
- try:
- self.fshelper.getPathForOID(oid, create=True)
- targetname = self.fshelper.getBlobFilename(oid, serial)
- rename_or_copy_blob(blobfilename, targetname)
+ def _blob_tpc_abort(self):
+ """Blob cleanup to be called from subclass tpc_abort
+ """
+ while self.dirty_oids:
+ oid, serial = self.dirty_oids.pop()
+ clean = self.fshelper.getBlobFilename(oid, serial)
+ if os.path.exists(clean):
+ remove_committed(clean)
- # if oid already in there, something is really hosed.
- # The underlying storage should have complained anyway
- self.dirty_oids.append((oid, serial))
- finally:
- self._lock_release()
-
@non_overridable
- def storeBlob(self, oid, oldserial, data, blobfilename, version,
- transaction):
- """Stores data that has a BLOB attached."""
- assert not version, "Versions aren't supported."
- serial = self.store(oid, oldserial, data, '', transaction)
- self._storeblob(oid, serial, blobfilename)
-
- return self._tid
-
- @non_overridable
- def restoreBlob(self, oid, serial, data, blobfilename, prev_txn,
- transaction):
- """Write blob data already committed in a separate database
+ def _blob_tpc_finish(self):
+ """Blob cleanup to be called from subclass tpc_finish
"""
- self.restore(oid, serial, data, '', prev_txn, transaction)
- self._storeblob(oid, serial, blobfilename)
+ self.dirty_oids = []
- return self._tid
-
@non_overridable
def copyTransactionsFrom(self, other):
for trans in other.iterator():
@@ -647,22 +639,6 @@
self.tpc_finish(trans)
@non_overridable
- def blob_tpc_finish(self):
- """Blob cleanup to be called from subclass tpc_finish
- """
- self.dirty_oids = []
-
- @non_overridable
- def blob_tpc_abort(self):
- """Blob cleanup to be called from subclass tpc_abort
- """
- while self.dirty_oids:
- oid, serial = self.dirty_oids.pop()
- clean = self.fshelper.getBlobFilename(oid, serial)
- if os.path.exists(clean):
- remove_committed(clean)
-
- @non_overridable
def loadBlob(self, oid, serial):
"""Return the filename where the blob file can be found.
"""
@@ -679,7 +655,45 @@
else:
return BlobFile(blob_filename, 'r', blob)
+ @non_overridable
+ def restoreBlob(self, oid, serial, data, blobfilename, prev_txn,
+ transaction):
+ """Write blob data already committed in a separate database
+ """
+ self.restore(oid, serial, data, '', prev_txn, transaction)
+ self._blob_storeblob(oid, serial, blobfilename)
+ return self._tid
+
+ @non_overridable
+ def _blob_storeblob(self, oid, serial, blobfilename):
+ self._lock_acquire()
+ try:
+ self.fshelper.getPathForOID(oid, create=True)
+ targetname = self.fshelper.getBlobFilename(oid, serial)
+ rename_or_copy_blob(blobfilename, targetname)
+
+ # if oid already in there, something is really hosed.
+ # The underlying storage should have complained anyway
+ self.dirty_oids.append((oid, serial))
+ finally:
+ self._lock_release()
+
+ @non_overridable
+ def storeBlob(self, oid, oldserial, data, blobfilename, version,
+ transaction):
+ """Stores data that has a BLOB attached."""
+ assert not version, "Versions aren't supported."
+ serial = self.store(oid, oldserial, data, '', transaction)
+ self._blob_storeblob(oid, serial, blobfilename)
+
+ return self._tid
+
+ @non_overridable
+ def temporaryDirectory(self):
+ return self.fshelper.temp_dir
+
+
class BlobStorage(SpecificationDecoratorBase, BlobStorageMixin):
"""A storage to support blobs."""
@@ -696,7 +710,7 @@
def __init__(self, base_directory, storage, layout='automatic'):
# XXX Log warning if storage is ClientStorage
SpecificationDecoratorBase.__init__(self, storage)
- BlobStorageMixin.__init__(self, base_directory, layout)
+ self._blob_init(base_directory, layout)
try:
supportsUndo = storage.supportsUndo
except AttributeError:
@@ -722,7 +736,7 @@
# providing a _finish method because methods found on the proxied
# object aren't rebound to the proxy
getProxiedObject(self).tpc_finish(*arg, **kw)
- self.blob_tpc_finish()
+ self._blob_tpc_finish()
@non_overridable
def tpc_abort(self, *arg, **kw):
@@ -730,7 +744,7 @@
# providing an _abort method because methods found on the proxied object
# aren't rebound to the proxy
getProxiedObject(self).tpc_abort(*arg, **kw)
- self.blob_tpc_abort()
+ self._blob_tpc_abort()
@non_overridable
def _packUndoing(self, packtime, referencesf):
Modified: ZODB/branches/jim-dev/src/ZODB/tests/testFileStorage.py
===================================================================
--- ZODB/branches/jim-dev/src/ZODB/tests/testFileStorage.py 2008-12-11 23:16:10 UTC (rev 93920)
+++ ZODB/branches/jim-dev/src/ZODB/tests/testFileStorage.py 2008-12-11 23:18:23 UTC (rev 93921)
@@ -14,6 +14,7 @@
import os, unittest
import transaction
import ZODB.FileStorage
+import ZODB.tests.testblob
import ZODB.tests.util
import zope.testing.setupstack
from ZODB import POSException
@@ -558,6 +559,13 @@
suite.addTest(doctest.DocTestSuite(
setUp=zope.testing.setupstack.setUpDirectory,
tearDown=zope.testing.setupstack.tearDown))
+ suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
+ 'BlobFileStorage',
+ lambda name, blob_dir:
+ ZODB.FileStorage.FileStorage('%s.fs' % name, blob_dir=blob_dir),
+ test_blob_storage_recovery=True,
+ test_packing=True,
+ ))
return suite
if __name__=='__main__':
More information about the Zodb-checkins
mailing list