[Zodb-checkins] SVN: ZODB/trunk/src/ZODB/ Added locking support for use by storages.
Jim Fulton
jim at zope.com
Sat Oct 25 20:36:09 EDT 2008
Log message for revision 92554:
Added locking support for use by storages.
Added a working newTid that replaces the non-working newTimeStamp.
Removed WeakSet, which is moved to the transaction package.
Changed:
U ZODB/trunk/src/ZODB/tests/testUtils.py
U ZODB/trunk/src/ZODB/utils.py
A ZODB/trunk/src/ZODB/utils.txt
-=-
Modified: ZODB/trunk/src/ZODB/tests/testUtils.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testUtils.py 2008-10-26 00:36:06 UTC (rev 92553)
+++ ZODB/trunk/src/ZODB/tests/testUtils.py 2008-10-26 00:36:08 UTC (rev 92554)
@@ -16,6 +16,7 @@
import random
import unittest
from persistent import Persistent
+from zope.testing import doctest
NUM = 100
@@ -89,9 +90,7 @@
def test_suite():
- return unittest.makeSuite(TestUtils, 'check')
-
-if __name__ == "__main__":
- loader = unittest.TestLoader()
- loader.testMethodPrefix = "check"
- unittest.main(testLoader=loader)
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestUtils, 'check'))
+ suite.addTest(doctest.DocFileSuite('../utils.txt'))
+ return suite
Modified: ZODB/trunk/src/ZODB/utils.py
===================================================================
--- ZODB/trunk/src/ZODB/utils.py 2008-10-26 00:36:06 UTC (rev 92553)
+++ ZODB/trunk/src/ZODB/utils.py 2008-10-26 00:36:08 UTC (rev 92554)
@@ -31,17 +31,17 @@
'u64',
'U64',
'cp',
- 'newTimeStamp',
+ 'newTid',
'oid_repr',
'serial_repr',
'tid_repr',
'positive_id',
'readable_tid_repr',
- 'WeakSet',
'DEPRECATED_ARGUMENT',
'deprecated37',
'deprecated38',
'get_pickle_metadata',
+ 'locked',
]
# A unique marker to give as the default value for a deprecated argument.
@@ -113,14 +113,12 @@
write(data)
length -= len(data)
-def newTimeStamp(old=None,
- TimeStamp=TimeStamp,
- time=time.time, gmtime=time.gmtime):
- t = time()
- ts = TimeStamp(gmtime(t)[:5]+(t%60,))
+def newTid(old):
+ t = time.time()
+ ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
if old is not None:
- return ts.laterThan(old)
- return ts
+ ts = ts.laterThan(TimeStamp(old))
+ return `ts`
def oid_repr(oid):
@@ -223,75 +221,52 @@
classname = ''
return modname, classname
-# A simple implementation of weak sets, supplying just enough of Python's
-# sets.Set interface for our needs.
+def mktemp(dir=None):
+ """Create a temp file, known by name, in a semi-secure manner."""
+ handle, filename = mkstemp(dir=dir)
+ os.close(handle)
+ return filename
-class WeakSet(object):
- """A set of objects that doesn't keep its elements alive.
+class Locked(object):
- The objects in the set must be weakly referencable.
- The objects need not be hashable, and need not support comparison.
- Two objects are considered to be the same iff their id()s are equal.
+ def __init__(self, func, inst=None, class_=None, preconditions=()):
+ self.im_func = func
+ self.im_self = inst
+ self.im_class = class_
+ self.preconditions = preconditions
- When the only references to an object are weak references (including
- those from WeakSets), the object can be garbage-collected, and
- will vanish from any WeakSets it may be a member of at that time.
- """
+ def __get__(self, inst, class_):
+ return self.__class__(self.im_func, inst, class_, self.preconditions)
- def __init__(self):
- # Map id(obj) to obj. By using ids as keys, we avoid requiring
- # that the elements be hashable or comparable.
- self.data = weakref.WeakValueDictionary()
+ def __call__(self, *args, **kw):
+ inst = self.im_self
+ if inst is None:
+ inst = args[0]
+ func = self.im_func.__get__(self.im_self, self.im_class)
- def __len__(self):
- return len(self.data)
+ inst._lock_acquire()
+ try:
+ for precondition in self.preconditions:
+ if not precondition(inst):
+ raise AssertionError(
+ "Failed precondition: ",
+ precondition.__doc__.strip())
+
+ return func(*args, **kw)
+ finally:
+ inst._lock_release()
- def __contains__(self, obj):
- return id(obj) in self.data
+class locked(object):
- # Same as a Set, add obj to the collection.
- def add(self, obj):
- self.data[id(obj)] = obj
+ def __init__(self, *preconditions):
+ self.preconditions = preconditions
- # Same as a Set, remove obj from the collection, and raise
- # KeyError if obj not in the collection.
- def remove(self, obj):
- del self.data[id(obj)]
+ def __get__(self, inst, class_):
+ # We didn't get any preconditions, so we have a single "precondition",
+ # which is actually the function to call.
+ func, = self.preconditions
+ return Locked(func, inst, class_)
- # f is a one-argument function. Execute f(elt) for each elt in the
- # set. f's return value is ignored.
- def map(self, f):
- for wr in self.as_weakref_list():
- elt = wr()
- if elt is not None:
- f(elt)
-
- # Return a list of weakrefs to all the objects in the collection.
- # Because a weak dict is used internally, iteration is dicey (the
- # underlying dict may change size during iteration, due to gc or
- # activity from other threads). as_weakef_list() is safe.
- #
- # Something like this should really be a method of Python's weak dicts.
- # If we invoke self.data.values() instead, we get back a list of live
- # objects instead of weakrefs. If gc occurs while this list is alive,
- # all the objects move to an older generation (because they're strongly
- # referenced by the list!). They can't get collected then, until a
- # less frequent collection of the older generation. Before then, if we
- # invoke self.data.values() again, they're still alive, and if gc occurs
- # while that list is alive they're all moved to yet an older generation.
- # And so on. Stress tests showed that it was easy to get into a state
- # where a WeakSet grows without bounds, despite that almost all its
- # elements are actually trash. By returning a list of weakrefs instead,
- # we avoid that, although the decision to use weakrefs is now# very
- # visible to our clients.
- def as_weakref_list(self):
- # We're cheating by breaking into the internals of Python's
- # WeakValueDictionary here (accessing its .data attribute).
- return self.data.data.values()
-
-
-def mktemp(dir=None):
- """Create a temp file, known by name, in a semi-secure manner."""
- handle, filename = mkstemp(dir=dir)
- os.close(handle)
- return filename
+ def __call__(self, func):
+ return Locked(func, preconditions=self.preconditions)
+
Added: ZODB/trunk/src/ZODB/utils.txt
===================================================================
--- ZODB/trunk/src/ZODB/utils.txt (rev 0)
+++ ZODB/trunk/src/ZODB/utils.txt 2008-10-26 00:36:08 UTC (rev 92554)
@@ -0,0 +1,196 @@
+ZODB Utilits Module
+===================
+
+The ZODB.utils module provides a number of helpful, somewhat random
+:), utility functions.
+
+ >>> import ZODB.utils
+
+This document documents a few of them. Over time, it may document
+more.
+
+64-bit integers and strings
+---------------------------------
+
+ZODB uses 64-bit transaction ids that are typically represented as
+strings, but are sometimes manipulated as integers. Object ids are
+strings too and it is common to ise 64-bit strings that are just
+packed integers.
+
+Functions p64 and u64 pack and unpack integers as strings:
+
+ >>> ZODB.utils.p64(250347764455111456)
+ '\x03yi\xf7"\xa8\xfb '
+
+ >>> print ZODB.utils.u64('\x03yi\xf7"\xa8\xfb ')
+ 250347764455111456
+
+The contant z64 has zero packed as a 64-bit string:
+
+ >>> ZODB.utils.z64
+ '\x00\x00\x00\x00\x00\x00\x00\x00'
+
+Transaction id generation
+-------------------------
+
+Storages assign transaction ids as transactions are committed. These
+are based on UTC time, but must be strictly increasing. The
+newTid function akes this pretty easy.
+
+To see this work (in a predictable way), we'll first hack time.time:
+
+ >>> import time
+ >>> old_time = time.time
+ >>> time.time = lambda : 1224825068.12
+
+Now, if we ask for a new time stamp, we'll get one based on our faux
+time:
+
+ >>> tid = ZODB.utils.newTid(None)
+ >>> tid
+ '\x03yi\xf7"\xa54\x88'
+
+newTid requires an old tid as an argument. The old tid may be None, if
+we don't have a previous transaction id.
+
+This time is based on the current time, which we can see by converting
+it to a time stamp.
+
+ >>> import ZODB.TimeStamp
+ >>> print ZODB.TimeStamp.TimeStamp(tid)
+ 2008-10-24 05:11:08.120000
+
+To assure that we get a new tid that is later than the old, we can
+pass an existing tid. Let's pass the tid we just got.
+
+ >>> tid2 = ZODB.utils.newTid(tid)
+ >>> ZODB.utils.u64(tid), ZODB.utils.u64(tid2)
+ (250347764454864008L, 250347764454864009L)
+
+Here, since we called it at the same time, we got a time stamp that
+was only slightly larger than the previos one. Of course, at a later
+time, the time stamp we get will be based on the time:
+
+ >>> time.time = lambda : 1224825069.12
+ >>> tid = ZODB.utils.newTid(tid2)
+ >>> print ZODB.TimeStamp.TimeStamp(tid)
+ 2008-10-24 05:11:09.120000
+
+
+ >>> time.time = old_time
+
+
+Locking support
+---------------
+
+Storages are required to be thread safe. The locking descriptor helps
+automate that. It arranges for a lock to be acquired when a function
+is called and released when a function exits. To demonstrate this,
+we'll create a "lock" type that simply prints when it is called:
+
+ >>> class Lock:
+ ... def acquire(self):
+ ... print 'acquire'
+ ... def release(self):
+ ... print 'release'
+
+Now we'll demonstrate the descriptor:
+
+ >>> class C:
+ ... _lock = Lock()
+ ... _lock_acquire = _lock.acquire
+ ... _lock_release = _lock.release
+ ...
+ ... @ZODB.utils.locked
+ ... def meth(self, *args, **kw):
+ ... print 'meth', args, kw
+
+The descriptor expects the instance it wraps to have a '_lock
+attribute.
+
+ >>> C().meth(1, 2, a=3)
+ acquire
+ meth (1, 2) {'a': 3}
+ release
+
+.. Edge cases
+
+ We can get the method from the class:
+
+ >>> C.meth # doctest: +ELLIPSIS
+ <ZODB.utils.Locked object at ...>
+
+ >>> C.meth(C())
+ acquire
+ meth () {}
+ release
+
+ >>> class C2:
+ ... _lock = Lock()
+ ... _lock_acquire = _lock.acquire
+ ... _lock_release = _lock.release
+
+ >>> C.meth(C2()) # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ TypeError: unbound method meth() must be called with C instance
+ as first argument (got C2 instance instead)
+
+Preconditions
+-------------
+
+Often, we want to supply method preconditions. The locking descriptor
+supports optional method preconditions [1]_.
+
+ >>> class C:
+ ... def __init__(self):
+ ... _lock = Lock()
+ ... self._lock_acquire = _lock.acquire
+ ... self._lock_release = _lock.release
+ ... self._opened = True
+ ... self._transaction = None
+ ...
+ ... def opened(self):
+ ... """The object is open
+ ... """
+ ... print 'checking if open'
+ ... return self._opened
+ ...
+ ... def not_in_transaction(self):
+ ... """The object is not in a transaction
+ ... """
+ ... print 'checking if in a transaction'
+ ... return self._transaction is None
+ ...
+ ... @ZODB.utils.locked(opened, not_in_transaction)
+ ... def meth(self, *args, **kw):
+ ... print 'meth', args, kw
+
+ >>> c = C()
+ >>> c.meth(1, 2, a=3)
+ acquire
+ checking if open
+ checking if in a transaction
+ meth (1, 2) {'a': 3}
+ release
+
+ >>> c._transaction = 1
+ >>> c.meth(1, 2, a=3) # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ AssertionError:
+ ('Failed precondition: ', 'The object is not in a transaction')
+
+ >>> c._opened = False
+ >>> c.meth(1, 2, a=3) # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ AssertionError: ('Failed precondition: ', 'The object is open')
+
+
+.. [1] Arguably, preconditions should be handled via separate
+ descriptors, but for ZODB storages, almost all methods need to be
+ locked. Combining preconditions with locking provides both
+ efficiency and concise expressions. A more general-purpose
+ facility would almost certainly provide separate descriptors for
+ preconditions.
Property changes on: ZODB/trunk/src/ZODB/utils.txt
___________________________________________________________________
Name: svn:eol-style
+ native
More information about the Zodb-checkins
mailing list