[Zodb-checkins]
SVN: ZODB/branches/blob-merge-branch/src/ZODB/Blobs/ - cleanups
Christian Theune
ct at gocept.com
Fri Sep 22 02:53:30 EDT 2006
Log message for revision 70325:
- cleanups
- added method to get a file handle that can be used for reading blobs outside
of a transaction.
- added documentation
- switched and cleaned up data manager API, especially the commit protocol
Changed:
U ZODB/branches/blob-merge-branch/src/ZODB/Blobs/Blob.py
U ZODB/branches/blob-merge-branch/src/ZODB/Blobs/interfaces.py
-=-
Modified: ZODB/branches/blob-merge-branch/src/ZODB/Blobs/Blob.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZODB/Blobs/Blob.py 2006-09-22 06:50:36 UTC (rev 70324)
+++ ZODB/branches/blob-merge-branch/src/ZODB/Blobs/Blob.py 2006-09-22 06:53:25 UTC (rev 70325)
@@ -1,28 +1,47 @@
+##############################################################################
+#
+# Copyright (c) 2005-2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""The blob class and related utilities.
+"""
+__docformat__ = "reStructuredText"
+
import os
import time
import tempfile
import logging
-from zope.interface import implements
+import zope.interface
from ZODB.Blobs.interfaces import IBlob
from ZODB.Blobs.exceptions import BlobError
from ZODB import utils
import transaction
-from transaction.interfaces import IDataManager
+import transaction.interfaces
from persistent import Persistent
+
BLOB_SUFFIX = ".blob"
+
class Blob(Persistent):
- implements(IBlob)
+ zope.interface.implements(IBlob)
_p_blob_readers = 0
_p_blob_writers = 0
- _p_blob_uncommitted = None
- _p_blob_data = None
+ _p_blob_uncommitted = None # Filename of the uncommitted (dirty) data
+ _p_blob_data = None # Filename of the committed data
# All persistent object store a reference to their data manager, a database
# connection in the _p_jar attribute. So we are going to do the same with
@@ -36,19 +55,10 @@
_p_blob_transaction = None
def open(self, mode="r"):
- """ Returns a file(-like) object representing blob data. This
- method will either return the file object, raise a BlobError
- or an IOError. A file may be open for exclusive read any
- number of times, but may not be opened simultaneously for read
- and write during the course of a single transaction and may
- not be opened for simultaneous writes during the course of a
- single transaction. Additionally, the file handle which
- results from this method call is unconditionally closed at
- transaction boundaries and so may not be used across
- transactions. """
+ """Returns a file(-like) object representing blob data."""
tempdir = os.environ.get('ZODB_BLOB_TEMPDIR', tempfile.gettempdir())
-
+
result = None
if (mode.startswith("r") or mode=="U"):
@@ -93,39 +103,48 @@
raise IOError, 'invalid mode: %s ' % mode
if result is not None:
-
- # we register ourselves as a data manager with the
- # transaction machinery in order to be notified of
- # commit/vote/abort events. We do this because at
- # transaction boundaries, we need to fix up _p_ reference
- # counts that keep track of open readers and writers and
- # close any writable filehandles we've opened.
-
+ # We join the transaction with our own data manager in order to be
+ # notified of commit/vote/abort events. We do this because at
+ # transaction boundaries, we need to fix up _p_ reference counts
+ # that keep track of open readers and writers and close any
+ # writable filehandles we've opened.
if self._p_blob_manager is None:
- dm = BlobDataManager(self, result)
-
# Blobs need to always participate in transactions.
if self._p_jar is not None:
- # If we are connected to a database, then we register
- # with the transaction manager for that.
- self._p_jar.transaction_manager.get().register(dm)
+ # If we are connected to a database, then we use the
+ # transaction manager that belongs to this connection
+ tm = self._p_jar.transaction_manager
else:
# If we are not connected to a database, we check whether
# we have been given an explicit transaction manager
if self._p_blob_transaction:
- self._p_blob_transaction.get().register(dm)
+ tm = self._p_blob_transaction
else:
- # Otherwise we register with the default
+ # Otherwise we use the default
# transaction manager as an educated guess.
- transaction.get().register(dm)
+ tm = transaction.manager
+ # Create our datamanager and join he current transaction.
+ dm = BlobDataManager(self, result, tm)
+ tm.get().join(dm)
else:
- # each blob data manager should manage only the one blob
- # assert that this is the case and it is the correct blob
+ # Each blob data manager should manage only the one blob
+ # assigned to it. Assert that this is the case and it is the
+ # correct blob
assert self._p_blob_manager.blob is self
self._p_blob_manager.register_fh(result)
-
return result
+ def openDetached(self):
+ """Returns a file(-like) object in read mode that can be used
+ outside of transaction boundaries.
+
+ """
+ if self._current_filename() is None:
+ raise BlobError, "Blob does not exist."
+ if self._p_blob_writers != 0:
+ raise BlobError, "Already opened for writing."
+ return file(self._current_filename(), "rb")
+
# utility methods
def _current_filename(self):
@@ -158,12 +177,13 @@
# used by unit tests
return self._p_blob_readers, self._p_blob_writers
+
class BlobDataManager:
- """Special data manager to handle transaction boundaries for blobs.
+ """Special data managerto handle transaction boundaries for blobs.
Blobs need some special care-taking on transaction boundaries. As
- a) the ghost objects might get reused, the _p_ reader and writer
+ a) the ghost objects might get reused, the _p_reader and _p_writer
refcount attributes must be set to a consistent state
b) the file objects might get passed out of the thread/transaction
and must deny any relationship to the original blob.
@@ -172,69 +192,90 @@
"""
- implements(IDataManager)
+ zope.interface.implements(transaction.interfaces.IDataManager)
- def __init__(self, blob, filehandle):
+ def __init__(self, blob, filehandle, tm):
self.blob = blob
+ self.transaction_manager = tm
# we keep a weakref to the file handle because we don't want to
# keep it alive if all other references to it die (e.g. in the
# case it's opened without assigning it to a name).
self.fhrefs = utils.WeakSet()
self.register_fh(filehandle)
- self.subtransaction = False
self.sortkey = time.time()
+ # Blob specific methods
+
def register_fh(self, filehandle):
self.fhrefs.add(filehandle)
- def abort_sub(self, transaction):
- pass
+ def _remove_uncommitted_data(self):
+ self.blob._p_blob_clear()
+ self.fhrefs.map(lambda fhref: fhref.close())
+ if self.blob._p_blob_uncommitted is not None and \
+ os.path.exists(self.blob._p_blob_uncommitted):
+ os.unlink(self.blob._p_blob_uncommitted)
- def commit_sub(self, transaction):
+ # IDataManager
+
+ def tpc_begin(self, transaction):
pass
- def tpc_begin(self, transaction, subtransaction=False):
- self.subtransaction = subtransaction
-
def tpc_abort(self, transaction):
pass
def tpc_finish(self, transaction):
- self.subtransaction = False
+ pass
def tpc_vote(self, transaction):
pass
-
- def commit(self, object, transaction):
- if not self.subtransaction:
- self.blob._p_blob_clear() # clear all blob refcounts
- self.fhrefs.map(lambda fhref: fhref.close())
- def abort(self, object, transaction):
- if not self.subtransaction:
- self.blob._p_blob_clear()
- self.fhrefs.map(lambda fhref: fhref.close())
- if self.blob._p_blob_uncommitted is not None and \
- os.path.exists(self.blob._p_blob_uncommitted):
- os.unlink(self.blob._p_blob_uncommitted)
+ def commit(self, transaction):
+ if not self.prepared:
+ raise TypeError('Not prepared to commit')
+ self._checkTransaction(transaction)
+ self.transaction = None
+ self.prepared = False
+ self.blob._p_blob_clear()
+ self.fhrefs.map(lambda fhref: fhref.close())
+
+ def abort(self, transaction):
+ self._checkTransaction(transaction)
+ if self.transaction is not None:
+ self.transaction = None
+ self.prepared = False
+
+ self._remove_uncommitted_data()
+
def sortKey(self):
return self.sortkey
- def beforeCompletion(self, transaction):
- pass
+ def prepare(self, transaction):
+ if self.prepared:
+ raise TypeError('Already prepared')
+ self._checkTransaction(transaction)
+ self.prepared = True
+ self.transaction = transaction
+ self.state += self.delta
- def afterCompletion(self, transaction):
- pass
+ def _checkTransaction(self, transaction):
+ if (self.transaction is not None and
+ self.transaction is not transaction):
+ raise TypeError("Transaction missmatch",
+ transaction, self.transaction)
+
class BlobFile(file):
-
- """ A BlobFile is a file that can be used within a transaction
- boundary; a BlobFile is just a Python file object, we only
- override methods which cause a change to blob data in order to
- call methods on our 'parent' persistent blob object signifying
- that the change happened. """
+ """A BlobFile that holds a file handle to actual blob data.
+ It is a file that can be used within a transaction boundary; a BlobFile is
+ just a Python file object, we only override methods which cause a change to
+ blob data in order to call methods on our 'parent' persistent blob object
+ signifying that the change happened.
+
+ """
+
# XXX these files should be created in the same partition as
# the storage later puts them to avoid copying them ...
@@ -271,15 +312,17 @@
# muddying the code needlessly.
self.close()
+
logger = logging.getLogger('ZODB.Blobs')
_pid = str(os.getpid())
+
def log(msg, level=logging.INFO, subsys=_pid, exc_info=False):
message = "(%s) %s" % (subsys, msg)
logger.log(level, message, exc_info=exc_info)
+
class FilesystemHelper:
-
# Storages that implement IBlobStorage can choose to use this
# helper class to generate and parse blob filenames. This is not
# a set-in-stone interface for all filesystem operations dealing
@@ -297,34 +340,41 @@
level=logging.INFO)
def isSecure(self, path):
- """ Ensure that (POSIX) path mode bits are 0700 """
+ """Ensure that (POSIX) path mode bits are 0700."""
return (os.stat(path).st_mode & 077) != 0
def checkSecure(self):
if not self.isSecure(self.base_dir):
log('Blob dir %s has insecure mode setting' % self.base_dir,
- level=logging.WARNING)
+ level=logging.WARNING)
def getPathForOID(self, oid):
- """ Given an OID, return the path on the filesystem where
- the blob data relating to that OID is stored """
+ """Given an OID, return the path on the filesystem where
+ the blob data relating to that OID is stored.
+
+ """
return os.path.join(self.base_dir, utils.oid_repr(oid))
def getBlobFilename(self, oid, tid):
- """ Given an oid and a tid, return the full filename of the
- 'committed' blob file related to that oid and tid. """
+ """Given an oid and a tid, return the full filename of the
+ 'committed' blob file related to that oid and tid.
+
+ """
oid_path = self.getPathForOID(oid)
filename = "%s%s" % (utils.tid_repr(tid), BLOB_SUFFIX)
return os.path.join(oid_path, filename)
def blob_mkstemp(self, oid, tid):
- """ Given an oid and a tid, return a temporary file descriptor
- and a related filename. The file is guaranteed to exist on
- the same partition as committed data, which is important for
- being able to rename the file without a copy operation. The
- directory in which the file will be placed, which is the
- return value of self.getPathForOID(oid), must exist before
- this method may be called successfully."""
+ """Given an oid and a tid, return a temporary file descriptor
+ and a related filename.
+
+ The file is guaranteed to exist on the same partition as committed
+ data, which is important for being able to rename the file without a
+ copy operation. The directory in which the file will be placed, which
+ is the return value of self.getPathForOID(oid), must exist before this
+ method may be called successfully.
+
+ """
oidpath = self.getPathForOID(oid)
fd, name = tempfile.mkstemp(suffix='.tmp', prefix=utils.tid_repr(tid),
dir=oidpath)
@@ -335,6 +385,7 @@
If the filename cannot be recognized as a blob filename, (None, None)
is returned.
+
"""
if not filename.endswith(BLOB_SUFFIX):
return None, None
@@ -347,8 +398,10 @@
return oid, serial
def getOIDsForSerial(self, search_serial):
- """ Return all oids related to a particular tid that exist in
- blob data """
+ """Return all oids related to a particular tid that exist in
+ blob data.
+
+ """
oids = []
base_dir = self.base_dir
for oidpath in os.listdir(base_dir):
@@ -358,4 +411,3 @@
if search_serial == serial:
oids.append(oid)
return oids
-
Modified: ZODB/branches/blob-merge-branch/src/ZODB/Blobs/interfaces.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZODB/Blobs/interfaces.py 2006-09-22 06:50:36 UTC (rev 70324)
+++ ZODB/branches/blob-merge-branch/src/ZODB/Blobs/interfaces.py 2006-09-22 06:53:25 UTC (rev 70325)
@@ -10,6 +10,17 @@
mode: Mode to open the file with. Possible values: r,w,r+,a
"""
+ def openDetached():
+ """Returns a file(-like) object in read mode that can be used
+ outside of transaction boundaries.
+
+ The file handle returned by this method is read-only and at the
+ beginning of the file.
+
+ The handle is not attached to the blob and can be used outside of a
+ transaction.
+ """
+
# XXX need a method to initialize the blob from the storage
# this means a) setting the _p_blob_data filename and b) putting
# the current data in that file
More information about the Zodb-checkins
mailing list