[Zodb-checkins] SVN: ZODB/branches/jim-dev/src/ZODB/ checkpoint

Jim Fulton jim at zope.com
Thu Dec 11 18:18:23 EST 2008


Log message for revision 93921:
  checkpoint

Changed:
  U   ZODB/branches/jim-dev/src/ZODB/FileStorage/FileStorage.py
  U   ZODB/branches/jim-dev/src/ZODB/blob.py
  U   ZODB/branches/jim-dev/src/ZODB/tests/testFileStorage.py

-=-
Modified: ZODB/branches/jim-dev/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/branches/jim-dev/src/ZODB/FileStorage/FileStorage.py	2008-12-11 23:16:10 UTC (rev 93920)
+++ ZODB/branches/jim-dev/src/ZODB/FileStorage/FileStorage.py	2008-12-11 23:18:23 UTC (rev 93921)
@@ -16,34 +16,36 @@
 $Revision: 1.16 $
 """
 
+from cPickle import Pickler, Unpickler, loads
+from persistent.TimeStamp import TimeStamp
+from struct import pack, unpack
+from types import StringType
+from zc.lockfile import LockFile
+from ZODB.FileStorage.format import CorruptedDataError
+from ZODB.FileStorage.format import FileStorageFormatter, DataHeader
+from ZODB.FileStorage.format import TRANS_HDR, TRANS_HDR_LEN
+from ZODB.FileStorage.format import TxnHeader, DATA_HDR, DATA_HDR_LEN
+from ZODB.FileStorage.fspack import FileStoragePacker
+from ZODB.fsIndex import fsIndex
+from ZODB import BaseStorage, ConflictResolution, POSException
+from ZODB.loglevels import BLATHER
+from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
+from ZODB.utils import p64, u64, z64
+
 import base64
-from cPickle import Pickler, Unpickler, loads
 import errno
+import logging
 import os
 import sys
 import time
-import logging
-from types import StringType
-from struct import pack, unpack
+import ZODB.blob
+import ZODB.interfaces
+import zope.interface
+import ZODB.utils
 
 # Not all platforms have fsync
 fsync = getattr(os, "fsync", None)
 
-import zope.interface
-import ZODB.interfaces
-from ZODB import BaseStorage, ConflictResolution, POSException
-from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
-from persistent.TimeStamp import TimeStamp
-from zc.lockfile import LockFile
-from ZODB.utils import p64, u64, cp, z64
-from ZODB.FileStorage.fspack import FileStoragePacker
-from ZODB.FileStorage.format import FileStorageFormatter, DataHeader
-from ZODB.FileStorage.format import TxnHeader, DATA_HDR, DATA_HDR_LEN
-from ZODB.FileStorage.format import TRANS_HDR, TRANS_HDR_LEN
-from ZODB.FileStorage.format import CorruptedDataError
-from ZODB.loglevels import BLATHER
-from ZODB.fsIndex import fsIndex
-
 packed_version = "FS21"
 
 logger = logging.getLogger('ZODB.FileStorage')
@@ -86,9 +88,12 @@
     def __init__(self, afile):
         self._file = afile
 
-class FileStorage(BaseStorage.BaseStorage,
-                  ConflictResolution.ConflictResolvingStorage,
-                  FileStorageFormatter):
+class FileStorage(
+    FileStorageFormatter,
+    ZODB.blob.BlobStorageMixin,
+    ConflictResolution.ConflictResolvingStorage,
+    BaseStorage.BaseStorage,
+    ):
 
     zope.interface.implements(
         ZODB.interfaces.IStorage,
@@ -102,7 +107,7 @@
     _pack_is_in_progress = False
 
     def __init__(self, file_name, create=False, read_only=False, stop=None,
-                 quota=None, pack_gc=True, packer=None):
+                 quota=None, pack_gc=True, packer=None, blob_dir=None):
 
         if read_only:
             self._is_read_only = True
@@ -198,6 +203,15 @@
 
         self._quota = quota
 
+        self.blob_dir = blob_dir
+        if blob_dir:
+            self._blob_init(blob_dir)
+            zope.interface.alsoProvides(self,
+                                        ZODB.interfaces.IBlobStorageRestoreable)
+        else:
+            self._blob_init_no_blobs()
+        
+
     def _initIndex(self, index, tindex):
         self._index=index
         self._tindex=tindex
@@ -654,7 +668,7 @@
                 h.descr = descr
                 h.ext = ext
                 self._file.write(h.asString())
-                cp(self._tfile, self._file, dlen)
+                ZODB.utils.cp(self._tfile, self._file, dlen)
                 self._file.write(p64(tl))
                 self._file.flush()
             except:
@@ -695,11 +709,13 @@
         self._pos = self._nextpos
         self._index.update(self._tindex)
         self._ltid = tid
+        self._blob_tpc_finish()
 
     def _abort(self):
         if self._nextpos:
             self._file.truncate(self._pos)
             self._nextpos=0
+            self._blob_tpc_abort()
 
     def _undoDataInfo(self, oid, pos, tpos):
         """Return the tid, data pointer, and data for the oid record at pos
@@ -920,6 +936,18 @@
                 # Don't fail right away. We may be redeemed later!
                 failures[h.oid] = v
             else:
+
+                if self.blob_dir and not p and prev:
+                    up, userial = self._loadBackTxn(h.oid, prev)
+                    if ZODB.blob.is_blob_record(up):
+                        # We're undoing a blob modification operation.
+                        # We have to copy the blob data
+                        tmp = ZODB.utils.mktemp(dir=self.fshelper.temp_dir)
+                        ZODB.utils.cp(
+                            self.openCommittedBlobFile(h.oid, userial),
+                            open(tmp, 'wb'))
+                        self._blob_storeblob(h.oid, self._tid, tmp)
+                
                 new = DataHeader(h.oid, self._tid, ipos, otloc, 0, len(p))
 
                 # TODO:  This seek shouldn't be necessary, but some other
@@ -1469,7 +1497,7 @@
                                name, oname)
                 o = open(oname,'wb')
                 file.seek(pos)
-                cp(file, o, file_size-pos)
+                ZODB.utils.cp(file, o, file_size-pos)
                 o.close()
                 break
     except:

Modified: ZODB/branches/jim-dev/src/ZODB/blob.py
===================================================================
--- ZODB/branches/jim-dev/src/ZODB/blob.py	2008-12-11 23:16:10 UTC (rev 93920)
+++ ZODB/branches/jim-dev/src/ZODB/blob.py	2008-12-11 23:18:23 UTC (rev 93921)
@@ -483,7 +483,15 @@
                 continue
             yield oid, path
 
+class NoBlobsFileSystemHelper:
 
+    @property
+    def temp_dir(self):
+        raise TypeError("Blobs are not supported")
+
+    getPathForOID = getBlobFilenamem = temp_dir
+
+
 class BlobStorageError(Exception):
     """The blob storage encountered an invalid state."""
 
@@ -577,50 +585,34 @@
 
     zope.interface.implements(ZODB.interfaces.IBlobStorage)
 
-    def __init__(self, blob_dir, layout='automatic'):
+    @non_overridable
+    def _blob_init(self, blob_dir, layout='automatic'):
         # XXX Log warning if storage is ClientStorage
         self.fshelper = FilesystemHelper(blob_dir, layout)
         self.fshelper.create()
         self.fshelper.checkSecure()
         self.dirty_oids = []
 
-    def temporaryDirectory(self):
-        return self.fshelper.temp_dir
+    def _blob_init_no_blobs(self):
+        self.fshelper = NoBlobsFileSystemHelper()
+        self.dirty_oids = []
 
     @non_overridable
-    def _storeblob(self, oid, serial, blobfilename):
-        self._lock_acquire()
-        try:
-            self.fshelper.getPathForOID(oid, create=True)
-            targetname = self.fshelper.getBlobFilename(oid, serial)
-            rename_or_copy_blob(blobfilename, targetname)
+    def _blob_tpc_abort(self):
+        """Blob cleanup to be called from subclass tpc_abort
+        """
+        while self.dirty_oids:
+            oid, serial = self.dirty_oids.pop()
+            clean = self.fshelper.getBlobFilename(oid, serial)
+            if os.path.exists(clean):
+                remove_committed(clean)
 
-            # if oid already in there, something is really hosed.
-            # The underlying storage should have complained anyway
-            self.dirty_oids.append((oid, serial))
-        finally:
-            self._lock_release()
-            
     @non_overridable
-    def storeBlob(self, oid, oldserial, data, blobfilename, version,
-                  transaction):
-        """Stores data that has a BLOB attached."""
-        assert not version, "Versions aren't supported."
-        serial = self.store(oid, oldserial, data, '', transaction)
-        self._storeblob(oid, serial, blobfilename)
-
-        return self._tid
-
-    @non_overridable
-    def restoreBlob(self, oid, serial, data, blobfilename, prev_txn,
-                    transaction):
-        """Write blob data already committed in a separate database
+    def _blob_tpc_finish(self):
+        """Blob cleanup to be called from subclass tpc_finish
         """
-        self.restore(oid, serial, data, '', prev_txn, transaction)
-        self._storeblob(oid, serial, blobfilename)
+        self.dirty_oids = []
 
-        return self._tid
-
     @non_overridable
     def copyTransactionsFrom(self, other):
         for trans in other.iterator():
@@ -647,22 +639,6 @@
             self.tpc_finish(trans)
 
     @non_overridable
-    def blob_tpc_finish(self):
-        """Blob cleanup to be called from subclass tpc_finish
-        """
-        self.dirty_oids = []
-
-    @non_overridable
-    def blob_tpc_abort(self):
-        """Blob cleanup to be called from subclass tpc_abort
-        """
-        while self.dirty_oids:
-            oid, serial = self.dirty_oids.pop()
-            clean = self.fshelper.getBlobFilename(oid, serial)
-            if os.path.exists(clean):
-                remove_committed(clean)
-
-    @non_overridable
     def loadBlob(self, oid, serial):
         """Return the filename where the blob file can be found.
         """
@@ -679,7 +655,45 @@
         else:
             return BlobFile(blob_filename, 'r', blob)
 
+    @non_overridable
+    def restoreBlob(self, oid, serial, data, blobfilename, prev_txn,
+                    transaction):
+        """Write blob data already committed in a separate database
+        """
+        self.restore(oid, serial, data, '', prev_txn, transaction)
+        self._blob_storeblob(oid, serial, blobfilename)
 
+        return self._tid
+
+    @non_overridable
+    def _blob_storeblob(self, oid, serial, blobfilename):
+        self._lock_acquire()
+        try:
+            self.fshelper.getPathForOID(oid, create=True)
+            targetname = self.fshelper.getBlobFilename(oid, serial)
+            rename_or_copy_blob(blobfilename, targetname)
+
+            # if oid already in there, something is really hosed.
+            # The underlying storage should have complained anyway
+            self.dirty_oids.append((oid, serial))
+        finally:
+            self._lock_release()
+            
+    @non_overridable
+    def storeBlob(self, oid, oldserial, data, blobfilename, version,
+                  transaction):
+        """Stores data that has a BLOB attached."""
+        assert not version, "Versions aren't supported."
+        serial = self.store(oid, oldserial, data, '', transaction)
+        self._blob_storeblob(oid, serial, blobfilename)
+
+        return self._tid
+
+    @non_overridable
+    def temporaryDirectory(self):
+        return self.fshelper.temp_dir
+
+
 class BlobStorage(SpecificationDecoratorBase, BlobStorageMixin):
     """A storage to support blobs."""
 
@@ -696,7 +710,7 @@
     def __init__(self, base_directory, storage, layout='automatic'):
         # XXX Log warning if storage is ClientStorage
         SpecificationDecoratorBase.__init__(self, storage)
-        BlobStorageMixin.__init__(self, base_directory, layout)
+        self._blob_init(base_directory, layout)
         try:
             supportsUndo = storage.supportsUndo
         except AttributeError:
@@ -722,7 +736,7 @@
         # providing a _finish method because methods found on the proxied 
         # object aren't rebound to the proxy
         getProxiedObject(self).tpc_finish(*arg, **kw)
-        self.blob_tpc_finish()
+        self._blob_tpc_finish()
 
     @non_overridable
     def tpc_abort(self, *arg, **kw):
@@ -730,7 +744,7 @@
         # providing an _abort method because methods found on the proxied object
         # aren't rebound to the proxy
         getProxiedObject(self).tpc_abort(*arg, **kw)
-        self.blob_tpc_abort()
+        self._blob_tpc_abort()
 
     @non_overridable
     def _packUndoing(self, packtime, referencesf):

Modified: ZODB/branches/jim-dev/src/ZODB/tests/testFileStorage.py
===================================================================
--- ZODB/branches/jim-dev/src/ZODB/tests/testFileStorage.py	2008-12-11 23:16:10 UTC (rev 93920)
+++ ZODB/branches/jim-dev/src/ZODB/tests/testFileStorage.py	2008-12-11 23:18:23 UTC (rev 93921)
@@ -14,6 +14,7 @@
 import os, unittest
 import transaction
 import ZODB.FileStorage
+import ZODB.tests.testblob
 import ZODB.tests.util
 import zope.testing.setupstack
 from ZODB import POSException
@@ -558,6 +559,13 @@
     suite.addTest(doctest.DocTestSuite(
         setUp=zope.testing.setupstack.setUpDirectory,
         tearDown=zope.testing.setupstack.tearDown))
+    suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
+        'BlobFileStorage',
+        lambda name, blob_dir:
+        ZODB.FileStorage.FileStorage('%s.fs' % name, blob_dir=blob_dir),
+        test_blob_storage_recovery=True,
+        test_packing=True,
+        ))
     return suite
 
 if __name__=='__main__':



More information about the Zodb-checkins mailing list