[Zodb-checkins] SVN: ZODB/branches/shane-poll-invalidations/src/ZODB/ Replaced IStoragePollable with IMVCCStorage.
Shane Hathaway
shane at hathawaymix.org
Sat Apr 25 19:00:46 EDT 2009
Log message for revision 99494:
Replaced IStoragePollable with IMVCCStorage.
IMVCCStorage better describes the contract of an MVCC-providing
storage implementation. Changes:
- bind_connection(zodb_conn) became new_instance(). Now we
don't need a connection to get an instance.
- The propagate_invalidations flag went away. Now we check
IMVCCStorage.providedBy(storage).
- connection_closing() became sync(force=False).
- added the release() method, which releases the backend
database session without closing the storage.
These changes were prompted by Jim, who suggested a better way to
integrate ZODB with RelStorage.
Changed:
U ZODB/branches/shane-poll-invalidations/src/ZODB/Connection.py
U ZODB/branches/shane-poll-invalidations/src/ZODB/DB.py
U ZODB/branches/shane-poll-invalidations/src/ZODB/interfaces.py
-=-
Modified: ZODB/branches/shane-poll-invalidations/src/ZODB/Connection.py
===================================================================
--- ZODB/branches/shane-poll-invalidations/src/ZODB/Connection.py 2009-04-25 22:49:53 UTC (rev 99493)
+++ ZODB/branches/shane-poll-invalidations/src/ZODB/Connection.py 2009-04-25 23:00:46 UTC (rev 99494)
@@ -30,6 +30,7 @@
from persistent.interfaces import IPersistentDataManager
from ZODB.interfaces import IConnection
from ZODB.interfaces import IBlobStorage
+from ZODB.interfaces import IMVCCStorage
from ZODB.blob import Blob, rename_or_copy_blob
from transaction.interfaces import ISavepointDataManager
from transaction.interfaces import IDataManagerSavepoint
@@ -95,10 +96,13 @@
self.connections = {self._db.database_name: self}
storage = db.storage
- m = getattr(storage, 'bind_connection', None)
- if m is not None:
- # Use a storage instance bound to this connection.
- storage = m(self)
+ if IMVCCStorage.providedBy(storage):
+ # Use a connection-specific storage instance.
+ self._mvcc_storage = True
+ storage = storage.new_instance()
+ else:
+ self._mvcc_storage = False
+
self._normal_storage = self._storage = storage
self.new_oid = storage.new_oid
self._savepoint_storage = None
@@ -153,13 +157,6 @@
# in the cache on abort and in other connections on finish.
self._modified = []
- # Allow the storage to decide whether invalidations should
- # propagate between connections. If the storage provides MVCC
- # semantics, it is better to not propagate invalidations between
- # connections.
- self._propagate_invalidations = getattr(
- self._storage, 'propagate_invalidations', True)
-
# _invalidated queues invalidate messages delivered from the DB
# _inv_lock prevents one thread from modifying the set while
# another is processing invalidations. All the invalidations
@@ -190,10 +187,10 @@
# _conflicts).
self._conflicts = {}
- # If MVCC is enabled, then _mvcc is True and _txn_time stores
- # the upper bound on transactions visible to this connection.
- # That is, all object revisions must be written before _txn_time.
- # If it is None, then the current revisions are acceptable.
+ # _txn_time stores the upper bound on transactions visible to
+ # this connection. That is, all object revisions must be
+ # written before _txn_time. If it is None, then the current
+ # revisions are acceptable.
self._txn_time = None
# To support importFile(), implemented in the ExportImport base
@@ -306,10 +303,8 @@
if self._opened:
self.transaction_manager.unregisterSynch(self)
- # If the storage wants to know, tell it this connection is closing.
- m = getattr(self._storage, 'connection_closing', None)
- if m is not None:
- m()
+ if self._mvcc_storage:
+ self._storage.sync(force=False)
if primary:
for connection in self.connections.values():
@@ -339,8 +334,9 @@
def invalidate(self, tid, oids):
"""Notify the Connection that transaction 'tid' invalidated oids."""
- if not self._propagate_invalidations:
- # The storage disabled inter-connection invalidation.
+ if self._mvcc_storage:
+ # Inter-connection invalidation is not needed when the
+ # storage provides MVCC.
return
if self.before is not None:
# this is an historical connection. Invalidations are irrelevant.
@@ -479,13 +475,11 @@
self._registered_objects = []
self._creating.clear()
- def _poll_invalidations(self):
- """Poll and process object invalidations provided by the storage.
- """
- m = getattr(self._storage, 'poll_invalidations', None)
- if m is not None:
+ # Process pending invalidations.
+ def _flush_invalidations(self):
+ if self._mvcc_storage:
# Poll the storage for invalidations.
- invalidated = m()
+ invalidated = self._storage.poll_invalidations()
if invalidated is None:
# special value: the transaction is so old that
# we need to flush the whole cache.
@@ -493,9 +487,6 @@
elif invalidated:
self._cache.invalidate(invalidated)
- # Process pending invalidations.
- def _flush_invalidations(self):
- self._poll_invalidations()
self._inv_lock.acquire()
try:
# Non-ghostifiable objects may need to read when they are
Modified: ZODB/branches/shane-poll-invalidations/src/ZODB/DB.py
===================================================================
--- ZODB/branches/shane-poll-invalidations/src/ZODB/DB.py 2009-04-25 22:49:53 UTC (rev 99493)
+++ ZODB/branches/shane-poll-invalidations/src/ZODB/DB.py 2009-04-25 23:00:46 UTC (rev 99494)
@@ -35,6 +35,7 @@
from zope.interface import implements
from ZODB.interfaces import IDatabase
+from ZODB.interfaces import IMVCCStorage
import BTrees.OOBTree
import transaction
@@ -425,28 +426,32 @@
DeprecationWarning, 2)
storage.tpc_vote = lambda *args: None
+ if IMVCCStorage.providedBy(storage):
+ temp_storage = storage.new_instance()
+ else:
+ temp_storage = storage
try:
- storage.load(z64, '')
- except KeyError:
- # Create the database's root in the storage if it doesn't exist
- from persistent.mapping import PersistentMapping
- root = PersistentMapping()
- # Manually create a pickle for the root to put in the storage.
- # The pickle must be in the special ZODB format.
- file = cStringIO.StringIO()
- p = cPickle.Pickler(file, 1)
- p.dump((root.__class__, None))
- p.dump(root.__getstate__())
- t = transaction.Transaction()
- t.description = 'initial database creation'
- storage.tpc_begin(t)
- storage.store(z64, None, file.getvalue(), '', t)
- storage.tpc_vote(t)
- storage.tpc_finish(t)
- if hasattr(storage, 'connection_closing'):
- # Let the storage release whatever resources it used for loading
- # the root object.
- storage.connection_closing()
+ try:
+ temp_storage.load(z64, '')
+ except KeyError:
+ # Create the database's root in the storage if it doesn't exist
+ from persistent.mapping import PersistentMapping
+ root = PersistentMapping()
+ # Manually create a pickle for the root to put in the storage.
+ # The pickle must be in the special ZODB format.
+ file = cStringIO.StringIO()
+ p = cPickle.Pickler(file, 1)
+ p.dump((root.__class__, None))
+ p.dump(root.__getstate__())
+ t = transaction.Transaction()
+ t.description = 'initial database creation'
+ temp_storage.tpc_begin(t)
+ temp_storage.store(z64, None, file.getvalue(), '', t)
+ temp_storage.tpc_vote(t)
+ temp_storage.tpc_finish(t)
+ finally:
+ if IMVCCStorage.providedBy(temp_storage):
+ temp_storage.release()
# Multi-database setup.
if databases is None:
Modified: ZODB/branches/shane-poll-invalidations/src/ZODB/interfaces.py
===================================================================
--- ZODB/branches/shane-poll-invalidations/src/ZODB/interfaces.py 2009-04-25 22:49:53 UTC (rev 99493)
+++ ZODB/branches/shane-poll-invalidations/src/ZODB/interfaces.py 2009-04-25 23:00:46 UTC (rev 99494)
@@ -953,46 +953,93 @@
# DB pass-through
-class IStoragePollable(Interface):
- """A storage that can be polled for changes."""
+class IMVCCStorage(IStorage):
+ """A storage that provides MVCC semantics internally.
- def bind_connection(connection):
- """Returns a storage instance to be used by the given Connection.
+ MVCC (multi-version concurrency control) means each user of a
+ database has a snapshot view of the database. The snapshot view
+ does not change, even if concurrent connections commit
+ transactions, until a transaction boundary. Relational databases
+ that support serializable transaction isolation provide MVCC.
- This method is optional. By implementing this method, a storage
- instance can maintain Connection-specific state.
+ Storages that implement IMVCCStorage, such as RelStorage, provide
+ MVCC semantics at the ZODB storage layer. When ZODB.Connection uses
+ a storage that implements IMVCCStorage, each connection uses a
+ connection-specific storage instance, and that storage instance
+ provides a snapshot of the database.
- If this method is not provided, all connections to the same database
- use the same storage instance (even across threads).
- """
+ By contrast, storages that do not implement IMVCCStorage, such as
+ FileStorage, rely on ZODB.Connection to provide MVCC semantics, so
+ in that case, one storage instance is shared by many
+ ZODB.Connections. Applications that use ZODB.Connection always have
+ a snapshot view of the database; IMVCCStorage only modifies which
+ layer of ZODB provides MVCC.
- propagate_invalidations = Attribute(
- """A boolean value indicating whether invalidations should propagate.
+ Furthermore, IMVCCStorage changes the way object invalidation
+ works. An essential feature of ZODB is the propagation of object
+ invalidation messages to keep in-memory caches up to date. Storages
+ like FileStorage and ZEO.ClientStorage send invalidation messages
+ to all other Connection instances at transaction commit time.
+ Storages that implement IMVCCStorage, on the other hand, expect the
+ ZODB.Connection to poll for a list of invalidated objects.
- ZODB normally sends invalidation notifications between
- Connection objects within a Python process. If this
- attribute is false, no such invalidations will be sent.
- Cross-connection invalidation should normally be enabled, but
- it adds unnecessary complexity to storages that expect the connection
- to poll for invalidations instead.
+ Certain methods of IMVCCStorage implementations open persistent
+ back end database sessions and retain the sessions even after the
+ method call finishes::
- If this attribute is not present, it is assumed to be true.
- """)
+ load
+ loadEx
+ loadSerial
+ loadBefore
+ store
+ restore
+ new_oid
+ history
+ tpc_begin
+ tpc_vote
+ tpc_abort
+ tpc_finish
- def connection_closing():
- """Notifies the storage that a connection is closing.
+ If you know that the storage instance will no longer be used after
+ calling any of these methods, you should call the release method to
+ release the persistent sessions. The persistent sessions will be
+ reopened as necessary if you call one of those methods again.
- This method is optional. This method is useful when
- bind_connection() provides Connection-specific storage instances.
- It lets the storage release resources.
+ Other storage methods open short lived back end sessions and close
+ the back end sessions before returning. These include::
+
+ __len__
+ getSize
+ undoLog
+ undo
+ pack
+ iterator
+
+ These methods do not provide MVCC semantics, so these methods
+ operate on the most current view of the database, rather than the
+ snapshot view that the other methods use.
+ """
+
+ def new_instance():
+ """Creates and returns another storage instance.
+
+ The returned instance provides IMVCCStorage and connects to the
+ same back-end database. The database state visible by the
+ instance will be a snapshot that varies independently of other
+ storage instances.
"""
+ def release():
+ """Release all persistent sessions used by this storage instance.
+
+ After this call, the storage instance can still be used;
+ calling methods that use persistent sessions will cause the
+ persistent sessions to be reopened.
+ """
+
def poll_invalidations():
"""Poll the storage for external changes.
- This method is optional. This method is useful when
- bind_connection() provides Connection-specific storage instances.
-
Returns either a sequence of OIDs that have changed, or None. When a
sequence is returned, the corresponding objects should be removed
from the ZODB in-memory cache. When None is returned, the storage is
@@ -1002,7 +1049,15 @@
In that case, the ZODB in-memory cache should be cleared.
"""
+ def sync(force=True):
+ """Updates the internal snapshot to the current state of the database.
+ If the force parameter is False, the storage may choose to
+ ignore this call. By ignoring this call, a storage can reduce
+ the frequency of database polls, thus reducing database load.
+ """
+
+
class IStorageCurrentRecordIteration(IStorage):
def record_iternext(next=None):
More information about the Zodb-checkins
mailing list