[Zodb-checkins] SVN: ZODB/trunk/src/ New Feature:
Jim Fulton
jim at zope.com
Fri Feb 12 17:48:06 EST 2010
Log message for revision 108958:
New Feature:
When transactions are aborted, new object ids allocated during the
transaction are saved and used in subsequent transactions. This can
help in situations where object ids are used as BTree keys and the
sequential allocation of object ids leads to conflict errors.
Changed:
U ZODB/trunk/src/CHANGES.txt
U ZODB/trunk/src/ZODB/Connection.py
U ZODB/trunk/src/ZODB/DB.py
A ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test
U ZODB/trunk/src/ZODB/tests/testConnection.py
U ZODB/trunk/src/ZODB/tests/testZODB.py
U ZODB/trunk/src/ZODB/tests/util.py
-=-
Modified: ZODB/trunk/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt 2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/CHANGES.txt 2010-02-12 22:48:05 UTC (rev 108958)
@@ -2,6 +2,17 @@
Change History
================
+3.10.0a2 (2010-??-??)
+=====================
+
+New Features
+------------
+
+- When transactions are aborted, new object ids allocated during the
+ transaction are saved and used in subsequent transactions. This can
+ help in situations where object ids are used as BTree keys and the
+ sequential allocation of object ids leads to conflict errors.
+
3.10.0a1 (2010-02-08)
=====================
Modified: ZODB/trunk/src/ZODB/Connection.py
===================================================================
--- ZODB/trunk/src/ZODB/Connection.py 2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/Connection.py 2010-02-12 22:48:05 UTC (rev 108958)
@@ -104,7 +104,7 @@
self._mvcc_storage = False
self._normal_storage = self._storage = storage
- self.new_oid = storage.new_oid
+ self.new_oid = db.new_oid
self._savepoint_storage = None
# Do we need to join a txn manager?
@@ -214,7 +214,7 @@
" added to a Connection.", obj)
elif obj._p_jar is None:
assert obj._p_oid is None
- oid = obj._p_oid = self._storage.new_oid()
+ oid = obj._p_oid = self.new_oid()
obj._p_jar = self
if self._added_during_commit is not None:
self._added_during_commit.append(obj)
@@ -426,6 +426,7 @@
if self._savepoint_storage is not None:
self._abort_savepoint()
+ self._invalidate_creating()
self._tpc_cleanup()
def _abort(self):
@@ -438,6 +439,7 @@
del self._added[oid]
del obj._p_jar
del obj._p_oid
+ self._db.save_oid(oid)
else:
# Note: If we invalidate a non-ghostifiable object
@@ -723,6 +725,7 @@
self._creating = {}
for oid in creating:
+ self._db.save_oid(oid)
o = self._cache.get(oid)
if o is not None:
del self._cache[oid]
Modified: ZODB/trunk/src/ZODB/DB.py
===================================================================
--- ZODB/trunk/src/ZODB/DB.py 2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/DB.py 2010-02-12 22:48:05 UTC (rev 108958)
@@ -384,7 +384,9 @@
historical_timeout=300,
database_name='unnamed',
databases=None,
- xrefs=True):
+ xrefs=True,
+ max_saved_oids=999,
+ ):
"""Create an object database.
:Parameters:
@@ -480,6 +482,9 @@
self._setupUndoMethods()
self.history = storage.history
+ self._saved_oids = []
+ self._max_saved_oids = max_saved_oids
+
def _setupUndoMethods(self):
storage = self.storage
try:
@@ -942,6 +947,19 @@
return ContextManager(self)
+ def save_oid(self, oid):
+ if len(self._saved_oids) < self._max_saved_oids:
+ self._saved_oids.append(oid)
+
+ def new_oid(self):
+ if self._saved_oids:
+ try:
+ return self._saved_oids.pop()
+ except IndexError:
+ pass # Hm, threads
+ return self.storage.new_oid()
+
+
class ContextManager:
"""PEP 343 context manager
"""
Added: ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test
===================================================================
--- ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test (rev 0)
+++ ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test 2010-02-12 22:48:05 UTC (rev 108958)
@@ -0,0 +1,64 @@
+New OIDs get reused if a transaction aborts
+===========================================
+
+Historical note:
+
+ An OID is a terrible thing to waste.
+
+ Seriously: sequential allocation of OIDs could cause problems when
+ OIDs are used as (or as the basis of) BTree keys. This happened
+ with Zope 3's intid utility where the object->id mapping uses an
+ object key based on the OID. We got frequent conflict errors
+ because, in a site with many users, many objects are added at the
+ same time and conficts happened when conflicting changes caused
+ bucket splits.
+
+ Reusing an earlier allocated, but discarded OID will allow retries
+ of transactions to work because they'll use earlier OIDs which won't
+ tend to conflict with newly allocated ones.
+
+If a transaction is aborted, new OIDs assigned in the transaction are
+saved and made available for later transactions.
+
+ >>> import ZODB.tests.util, transaction
+ >>> db = ZODB.tests.util.DB()
+ >>> tm1 = transaction.TransactionManager()
+ >>> conn1 = db.open(tm1)
+ >>> conn1.root.x = ZODB.tests.util.P()
+ >>> tm1.commit()
+ >>> conn1.root.x.x = ZODB.tests.util.P()
+ >>> conn1.root.y = 1
+ >>> tm2 = transaction.TransactionManager()
+ >>> conn2 = db.open(tm2)
+ >>> conn2.root.y = ZODB.tests.util.P()
+ >>> tm2.commit()
+
+We get a conflict when we try to commit the change to the first connection:
+
+ >>> tm1.commit() # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ConflictError: ...
+
+ >>> tm1.abort()
+
+When we try, we get the same oid we would have gotten on the first transaction:
+
+ >>> conn1.root.x.x = ZODB.tests.util.P()
+ >>> tm1.commit()
+ >>> conn1.root.x.x._p_oid
+ '\x00\x00\x00\x00\x00\x00\x00\x03'
+
+We see this more clearly when we use add to assign the oid before the commit:
+
+ >>> conn1.root.z = ZODB.tests.util.P()
+ >>> conn1.add(conn1.root.z)
+ >>> conn1.root.z._p_oid
+ '\x00\x00\x00\x00\x00\x00\x00\x04'
+
+ >>> tm1.abort()
+
+ >>> conn2.root.a = ZODB.tests.util.P()
+ >>> conn2.add(conn2.root.a)
+ >>> conn2.root.a._p_oid
+ '\x00\x00\x00\x00\x00\x00\x00\x04'
Property changes on: ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test
___________________________________________________________________
Added: svn:eol-style
+ native
Modified: ZODB/trunk/src/ZODB/tests/testConnection.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testConnection.py 2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/tests/testConnection.py 2010-02-12 22:48:05 UTC (rev 108958)
@@ -815,6 +815,7 @@
def __init__(self):
self.storage = StubStorage()
+ self.new_oid = self.storage.new_oid
classFactory = None
database_name = 'stubdatabase'
@@ -823,6 +824,8 @@
def invalidate(self, transaction, dict_with_oid_keys, connection):
pass
+ save_oid = lambda self, oid: None
+
def test_suite():
s = unittest.makeSuite(ConnectionDotAdd, 'check')
s.addTest(doctest.DocTestSuite())
Modified: ZODB/trunk/src/ZODB/tests/testZODB.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testZODB.py 2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/tests/testZODB.py 2010-02-12 22:48:05 UTC (rev 108958)
@@ -11,21 +11,22 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
+
+from persistent import Persistent
+from persistent.mapping import PersistentMapping
+from ZODB.POSException import ReadConflictError, ConflictError
+from ZODB.POSException import TransactionFailedError
+from ZODB.tests.warnhook import WarningsHook
+
+import doctest
+import transaction
import unittest
import warnings
-
import ZODB
import ZODB.FileStorage
import ZODB.MappingStorage
-from ZODB.POSException import ReadConflictError, ConflictError
-from ZODB.POSException import TransactionFailedError
-from ZODB.tests.warnhook import WarningsHook
import ZODB.tests.util
-from persistent import Persistent
-from persistent.mapping import PersistentMapping
-import transaction
-
class P(Persistent):
pass
@@ -631,7 +632,9 @@
self._p_jar = poisonedjar
def test_suite():
- return unittest.makeSuite(ZODBTests, 'check')
+ suite = unittest.makeSuite(ZODBTests, 'check')
+ suite.addTest(doctest.DocFileSuite('new_oids_get_reused_on_abort.test'))
+ return suite
if __name__ == "__main__":
unittest.main(defaultTest="test_suite")
Modified: ZODB/trunk/src/ZODB/tests/util.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/util.py 2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/tests/util.py 2010-02-12 22:48:05 UTC (rev 108958)
@@ -57,7 +57,7 @@
class P(persistent.Persistent):
- def __init__(self, name):
+ def __init__(self, name=None):
self.name = name
def __repr__(self):
More information about the Zodb-checkins
mailing list