[Zodb-checkins] CVS: ZODB4/src/zodb/storage/tests - base.py:1.9 basic.py:1.8 mt.py:1.6 packable.py:1.5 readonly.py:1.6 recovery.py:1.5 revision.py:1.6 synchronization.py:1.5 test_autopack.py:1.10 test_file.py:1.7 undo.py:1.7 undoversion.py:1.4 version.py:1.4
Barry Warsaw
barry@wooz.org
Thu, 13 Mar 2003 16:33:01 -0500
Update of /cvs-repository/ZODB4/src/zodb/storage/tests
In directory cvs.zope.org:/tmp/cvs-serv27419/src/zodb/storage/tests
Modified Files:
base.py basic.py mt.py packable.py readonly.py recovery.py
revision.py synchronization.py test_autopack.py test_file.py
undo.py undoversion.py version.py
Log Message:
> I believe we're ready to merge back to the head.
merging the opaque-pickles-branch back into the head
=== ZODB4/src/zodb/storage/tests/base.py 1.8 => 1.9 ===
--- ZODB4/src/zodb/storage/tests/base.py:1.8 Tue Feb 25 17:57:52 2003
+++ ZODB4/src/zodb/storage/tests/base.py Thu Mar 13 16:32:29 2003
@@ -23,14 +23,14 @@
import os
import sys
import errno
-import types
import shutil
import unittest
+from persistence import Persistent
from transaction import get_transaction
from zodb.db import DB
-from zodb.serialize import ConnectionObjectReader, ObjectWriter
+from zodb.serialize import ConnectionObjectReader, ObjectWriter, findrefs
from zodb.ztransaction import Transaction
from zodb.storage.tests.minpo import MinPO
from zodb.storage.base import ZERO, BerkeleyConfig
@@ -62,13 +62,13 @@
"""
d = {}
for arg in args:
- if isinstance(arg, types.StringType):
+ if isinstance(arg, str):
d[oid] = arg
elif arg is None:
pass
else:
for oid, serial in arg:
- if not isinstance(serial, types.StringType):
+ if not isinstance(serial, str):
raise serial # error from ZEO server
d[oid] = serial
return d
@@ -94,6 +94,20 @@
if err[0] != errno.ENOENT:
raise
+class C(Persistent):
+ pass
+
+
+class Jar:
+ def __init__(self, storage):
+ self._storage = storage
+
+ def newObjectId(self):
+ return self._storage.newObjectId()
+
+ def register(self, obj):
+ obj._p_oid = self.newObjectId()
+
class StorageTestBase(unittest.TestCase):
@@ -114,7 +128,7 @@
self._close()
def _dostore(self, oid=None, revid=None, data=None, version=None,
- already_pickled=0, user=None, description=None):
+ already_pickled=False, user=None, description=None):
"""Do a complete storage transaction. The defaults are:
- oid=None, ask the storage for a new oid
@@ -128,12 +142,17 @@
oid = self._storage.newObjectId()
if revid is None:
revid = ZERO
+ refs = []
if data is None:
data = MinPO(7)
- if type(data) == types.IntType:
+ if isinstance(data, int):
data = MinPO(data)
if not already_pickled:
- data = zodb_pickle(data)
+ data, refs = zodb_pickle(data)
+ elif isinstance(data, tuple):
+ data, refs = data
+ else:
+ refs = findrefs(data)
if version is None:
version = ''
# Begin the transaction
@@ -145,7 +164,7 @@
try:
self._storage.tpcBegin(t)
# Store an object
- r1 = self._storage.store(oid, revid, data, version, t)
+ r1 = self._storage.store(oid, revid, data, refs, version, t)
# Finish the transaction
r2 = self._storage.tpcVote(t)
revid = handle_serials(oid, r1, r2)
@@ -157,7 +176,7 @@
def _dostoreNP(self, oid=None, revid=None, data=None, version=None,
user=None, description=None):
- return self._dostore(oid, revid, data, version, already_pickled=1,
+ return self._dostore(oid, revid, data, version, already_pickled=True,
user=user, description=description)
# The following methods depend on optional storage features.
@@ -193,6 +212,45 @@
self._storage.tpcFinish(t)
return oids
+ # some helper functions for setting up a valid root so that
+ # the storage can be packed
+
+ def _initroot(self):
+ self._jar = jar = Jar(self._storage)
+ self._reader = ConnectionObjectReader(jar, {})
+ try:
+ self._storage.load(ZERO, '')
+ except KeyError:
+ root = self._newobj()
+ t = Transaction()
+ t.note("initial database creation")
+ self._storage.tpcBegin(t)
+ data, refs = zodb_pickle(root)
+ self._storage.store(ZERO, None, data, refs, '', t)
+ self._storage.tpcVote(t)
+ self._storage.tpcFinish(t)
+
+ def _newobj(self):
+ obj = C()
+ obj._p_jar = self._jar
+ return obj
+
+ def _linked_newobj(self):
+ # Create a new object and make sure the root has a reference to it.
+ # Returns the object, the root, and the revid of the root.
+ data, revid0 = self._storage.load(ZERO, '')
+ root = self._reader.getObject(data)
+ obj = self._newobj()
+ # Link the root object to the persistent object, in order to keep the
+ # persistent object alive. XXX Order here is important: an attribute
+ # on the root object must be set first, so that it gets oid 0, /then/
+ # the attribute on the obj can be set.
+ root.obj = obj
+ obj.value = 0
+ root.value = 0
+ root._p_jar = self._jar
+ revid0 = self._dostore(ZERO, revid=revid0, data=root)
+ return obj, root, revid0
class BerkeleyTestBase(StorageTestBase):
=== ZODB4/src/zodb/storage/tests/basic.py 1.7 => 1.8 ===
--- ZODB4/src/zodb/storage/tests/basic.py:1.7 Tue Feb 11 10:59:28 2003
+++ ZODB4/src/zodb/storage/tests/basic.py Thu Mar 13 16:32:29 2003
@@ -35,7 +35,7 @@
def checkIStorage(self):
self.assert_(IStorage.isImplementedBy(self._storage))
-
+
def checkDatabaseVersion(self):
version = "abcd"
self._storage.setVersion(version)
@@ -59,7 +59,7 @@
# different Transaction object than the one we've begun on.
self._storage.tpcBegin(t)
self.assertRaises(StorageTransactionError, self._storage.store,
- 0, 0, 0, 0, Transaction())
+ 0, 0, 0, 0, 0, Transaction())
try:
self._storage.abortVersion('dummy', Transaction())
@@ -76,7 +76,7 @@
assert 0, "Should have failed, invalid transaction."
self.assertRaises(StorageTransactionError, self._storage.store,
- 0, 1, 2, 3, Transaction())
+ 0, 1, 2, 3, 4, Transaction())
self._storage.tpcAbort(t)
def checkSerialIsNoneForInitialRevision(self):
@@ -86,8 +86,8 @@
self._storage.tpcBegin(txn)
# Use None for serial. Don't use _dostore() here because that coerces
# serial=None to serial=ZERO.
- r1 = self._storage.store(oid, None, zodb_pickle(MinPO(11)),
- '', txn)
+ data, refs = zodb_pickle(MinPO(11))
+ r1 = self._storage.store(oid, None, data, refs, '', txn)
r2 = self._storage.tpcVote(txn)
self._storage.tpcFinish(txn)
newrevid = handle_serials(oid, r1, r2)
@@ -133,7 +133,8 @@
oid = self._storage.newObjectId()
t = Transaction()
self._storage.tpcBegin(t)
- self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
+ data, refs = zodb_pickle(MinPO(5))
+ self._storage.store(oid, ZERO, data, refs, '', t)
# Now abort this transaction
self._storage.tpcAbort(t)
# Now start all over again
@@ -146,7 +147,8 @@
oid = self._storage.newObjectId()
t = Transaction()
self._storage.tpcBegin(t)
- self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
+ data, refs = zodb_pickle(MinPO(5))
+ self._storage.store(oid, ZERO, data, refs, '', t)
# Now abort this transaction
self._storage.tpcVote(t)
self._storage.tpcAbort(t)
@@ -191,7 +193,7 @@
tid = chr(42) * 8
self._storage.tpcBegin(t, tid)
oid = self._storage.newObjectId()
- data = zodb_pickle(MinPO(8))
- self._storage.store(oid, None, data, '', t)
+ data, refs = zodb_pickle(MinPO(8))
+ self._storage.store(oid, None, data, refs, '', t)
self._storage.tpcVote(t)
self._storage.tpcFinish(t)
=== ZODB4/src/zodb/storage/tests/mt.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/storage/tests/mt.py:1.5 Thu Mar 13 15:48:50 2003
+++ ZODB4/src/zodb/storage/tests/mt.py Thu Mar 13 16:32:29 2003
@@ -143,7 +143,7 @@
return oid
def dostore(self, i):
- data = zodb_pickle(MinPO((self.getName(), i)))
+ data, refs = zodb_pickle(MinPO((self.getName(), i)))
t = Transaction()
oid = self.oid()
self.pause()
@@ -152,7 +152,7 @@
self.pause()
# Always create a new object, signified by None for revid
- r1 = self.storage.store(oid, None, data, '', t)
+ r1 = self.storage.store(oid, None, data, refs, '', t)
self.pause()
r2 = self.storage.tpcVote(t)
=== ZODB4/src/zodb/storage/tests/packable.py 1.4 => 1.5 ===
--- ZODB4/src/zodb/storage/tests/packable.py:1.4 Wed Feb 5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/packable.py Thu Mar 13 16:32:29 2003
@@ -14,127 +14,20 @@
"""Run some tests relevant for storages that support pack()."""
import time
-from cStringIO import StringIO
-try:
- import cPickle
- pickle = cPickle
- #import cPickle as pickle
-except ImportError:
- import pickle
-
-from persistence import Persistent
from transaction import get_transaction
from zodb.db import DB
-from zodb.storage.base import ZERO
+from zodb.interfaces import ZERO
+from zodb.serialize import getDBRoot, ConnectionObjectReader
+from zodb.ztransaction import Transaction
from zodb.storage.tests.minpo import MinPO
+from zodb.storage.tests.base import zodb_unpickle
+from zodb.storage.fsdump import dump
-# This class is for the root object. It must not contain a getoid() method
-# (really, attribute). The persistent pickling machinery -- in the dumps()
-# function below -- will pickle Root objects as normal, but any attributes
-# which reference persistent Object instances will get pickled as persistent
-# ids, not as the object's state. This makes the referencesf stuff work,
-# because it pickle sniffs for persistent ids (so we have to get those
-# persistent ids into the root object's pickle).
-class Root:
- pass
-
-
-# This is the persistent Object class. Because it has a getoid() method, the
-# persistent pickling machinery -- in the dumps() function below -- will
-# pickle the oid string instead of the object's actual state. Yee haw, this
-# stuff is deep. ;)
-class Object:
- def __init__(self, oid):
- self._oid = oid
-
- def getoid(self):
- return self._oid
-
-class C(Persistent):
- pass
-
-
-# Here's where all the magic occurs. Sadly, the pickle module is a bit
-# underdocumented, but here's what happens: by setting the persistent_id
-# attribute to getpersid() on the pickler, that function gets called for every
-# object being pickled. By returning None when the object has no getoid
-# attribute, it signals pickle to serialize the object as normal. That's how
-# the Root instance gets pickled correctly. But, if the object has a getoid
-# attribute, then by returning that method's value, we tell pickle to
-# serialize the persistent id of the object instead of the object's state.
-# That sets the pickle up for proper sniffing by the referencesf machinery.
-# Fun, huh?
-def dumps(obj):
- def getpersid(obj):
- if hasattr(obj, 'getoid'):
- return obj.getoid()
- return None
- s = StringIO()
- p = pickle.Pickler(s)
- p.persistent_id = getpersid
- p.dump(obj)
- return s.getvalue()
-
-
-
-class PackableStorageBase:
- # We keep a cache of object ids to instances so that the unpickler can
- # easily return any persistent object.
- _cache = {}
-
- def _newobj(self):
- # This is a convenience method to create a new persistent Object
- # instance. It asks the storage for a new object id, creates the
- # instance with the given oid, populates the cache and returns the
- # object.
- oid = self._storage.newObjectId()
- obj = Object(oid)
- self._cache[obj.getoid()] = obj
- return obj
-
- def _makeloader(self):
- # This is the other side of the persistent pickling magic. We need a
- # custom unpickler to mirror our custom pickler above. By setting the
- # persistent_load function of the unpickler to self._cache.get(),
- # whenever a persistent id is unpickled, it will actually return the
- # Object instance out of the cache. As far as returning a function
- # with an argument bound to an instance attribute method, we do it
- # this way because it makes the code in the tests more succinct.
- #
- # BUT! Be careful in your use of loads() vs. pickle.loads(). loads()
- # should only be used on the Root object's pickle since it's the only
- # special one. All the Object instances should use pickle.loads().
- def loads(str, persfunc=self._cache.get):
- fp = StringIO(str)
- u = pickle.Unpickler(fp)
- u.persistent_load = persfunc
- return u.load()
- return loads
-
-
-
-class PackableStorage(PackableStorageBase):
- def _initroot(self):
- try:
- self._storage.load(ZERO, '')
- except KeyError:
- from persistence.dict import PersistentDict
- from zodb.ztransaction import Transaction
- file = StringIO()
- p = cPickle.Pickler(file, 1)
- p.dump((PersistentDict, None))
- p.dump(PersistentDict().__getstate__())
- t = Transaction()
- t.note("initial database creation")
- self._storage.tpcBegin(t)
- self._storage.store(ZERO, None, file.getvalue(), '', t)
- self._storage.tpcVote(t)
- self._storage.tpcFinish(t)
-
+class PackableStorage:
def checkPackEmptyStorage(self):
self._storage.pack(time.time())
@@ -152,30 +45,29 @@
raises = self.assertRaises
# Create a `persistent' object
obj = self._newobj()
- oid = obj.getoid()
obj.value = 1
+ oid = obj._p_oid
# Commit three different revisions
- revid1 = self._dostoreNP(oid, data=pickle.dumps(obj))
+ revid1 = self._dostore(oid, data=obj)
obj.value = 2
- revid2 = self._dostoreNP(oid, revid=revid1, data=pickle.dumps(obj))
+ revid2 = self._dostore(oid, revid=revid1, data=obj)
obj.value = 3
- revid3 = self._dostoreNP(oid, revid=revid2, data=pickle.dumps(obj))
+ revid3 = self._dostore(oid, revid=revid2, data=obj)
# Now make sure all three revisions can be extracted
data = self._storage.loadSerial(oid, revid1)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 1)
data = self._storage.loadSerial(oid, revid2)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 2)
data = self._storage.loadSerial(oid, revid3)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 3)
# Now pack all transactions; need to sleep a second to make
# sure that the pack time is greater than the last commit time.
- time.sleep(1)
+ now = int(time.time())
+ while now >= int(time.time()):
+ time.sleep(0.1)
self._storage.pack(time.time())
# All revisions of the object should be gone, since there is no
# reference from the root object to this object.
@@ -186,147 +78,132 @@
def checkPackJustOldRevisions(self):
eq = self.assertEqual
raises = self.assertRaises
- loads = self._makeloader()
- # Create a root object. This can't be an instance of Object,
- # otherwise the pickling machinery will serialize it as a persistent
- # id and not as an object that contains references (persistent ids) to
- # other objects.
- root = Root()
- # Create a persistent object, with some initial state
- obj = self._newobj()
- oid = obj.getoid()
- # Link the root object to the persistent object, in order to keep the
- # persistent object alive. Store the root object.
- root.obj = obj
- root.value = 0
- revid0 = self._dostoreNP(ZERO, data=dumps(root))
+ # Create a root object
+ self._initroot()
+ obj, root, revid0 = self._linked_newobj()
# Make sure the root can be retrieved
data, revid = self._storage.load(ZERO, '')
eq(revid, revid0)
- eq(loads(data).value, 0)
+ eq(zodb_unpickle(data).value, 0)
# Commit three different revisions of the other object
obj.value = 1
- revid1 = self._dostoreNP(oid, data=pickle.dumps(obj))
+ oid = obj._p_oid
+ revid1 = self._dostore(oid, data=obj)
obj.value = 2
- revid2 = self._dostoreNP(oid, revid=revid1, data=pickle.dumps(obj))
+ revid2 = self._dostore(oid, revid=revid1, data=obj)
obj.value = 3
- revid3 = self._dostoreNP(oid, revid=revid2, data=pickle.dumps(obj))
+ revid3 = self._dostore(oid, revid=revid2, data=obj)
# Now make sure all three revisions can be extracted
data = self._storage.loadSerial(oid, revid1)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 1)
data = self._storage.loadSerial(oid, revid2)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 2)
data = self._storage.loadSerial(oid, revid3)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 3)
# Now pack just revisions 1 and 2. The object's current revision
# should stay alive because it's pointed to by the root.
- time.sleep(1)
+ now = int(time.time())
+ while now >= int(time.time()):
+ time.sleep(0.1)
self._storage.pack(time.time())
- # Make sure the revisions are gone, but that object zero and revision
- # 3 are still there and correct
+ # Make sure the revisions are gone, but that the root object and
+ # revision 3 are still there and correct
data, revid = self._storage.load(ZERO, '')
eq(revid, revid0)
- eq(loads(data).value, 0)
+ eq(zodb_unpickle(data).value, 0)
raises(KeyError, self._storage.loadSerial, oid, revid1)
raises(KeyError, self._storage.loadSerial, oid, revid2)
data = self._storage.loadSerial(oid, revid3)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 3)
data, revid = self._storage.load(oid, '')
eq(revid, revid3)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 3)
def checkPackOnlyOneObject(self):
eq = self.assertEqual
raises = self.assertRaises
- loads = self._makeloader()
- # Create a root object. This can't be an instance of Object,
- # otherwise the pickling machinery will serialize it as a persistent
- # id and not as an object that contains references (persistent ids) to
- # other objects.
- root = Root()
+ # Create a root object.
+ self._initroot()
+ data, revid0 = self._storage.load(ZERO, '')
+ root = self._reader.getObject(data)
+ root.value = -1
+ root._p_jar = self._jar
# Create a persistent object, with some initial state
obj1 = self._newobj()
- oid1 = obj1.getoid()
+ obj1.value = -1
+ oid1 = obj1._p_oid
# Create another persistent object, with some initial state. Make
# sure its oid is greater than the first object's oid.
obj2 = self._newobj()
- oid2 = obj2.getoid()
+ obj2.value = -1
+ oid2 = obj2._p_oid
self.failUnless(oid2 > oid1)
# Link the root object to the persistent objects, in order to keep
# them alive. Store the root object.
root.obj1 = obj1
root.obj2 = obj2
root.value = 0
- revid0 = self._dostoreNP(ZERO, data=dumps(root))
+ revid0 = self._dostore(ZERO, data=root, revid=revid0)
# Make sure the root can be retrieved
data, revid = self._storage.load(ZERO, '')
eq(revid, revid0)
- eq(loads(data).value, 0)
+ eq(zodb_unpickle(data).value, 0)
# Commit three different revisions of the first object
obj1.value = 1
- revid1 = self._dostoreNP(oid1, data=pickle.dumps(obj1))
+ revid1 = self._dostore(oid1, data=obj1)
obj1.value = 2
- revid2 = self._dostoreNP(oid1, revid=revid1, data=pickle.dumps(obj1))
+ revid2 = self._dostore(oid1, revid=revid1, data=obj1)
obj1.value = 3
- revid3 = self._dostoreNP(oid1, revid=revid2, data=pickle.dumps(obj1))
+ revid3 = self._dostore(oid1, revid=revid2, data=obj1)
# Now make sure all three revisions can be extracted
data = self._storage.loadSerial(oid1, revid1)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid1)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 1)
data = self._storage.loadSerial(oid1, revid2)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid1)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 2)
data = self._storage.loadSerial(oid1, revid3)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid1)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 3)
# Now commit a revision of the second object
obj2.value = 11
- revid4 = self._dostoreNP(oid2, data=pickle.dumps(obj2))
+ revid4 = self._dostore(oid2, data=obj2)
# And make sure the revision can be extracted
data = self._storage.loadSerial(oid2, revid4)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid2)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 11)
# Now pack just revisions 1 and 2 of object1. Object1's current
# revision should stay alive because it's pointed to by the root, as
# should Object2's current revision.
- time.sleep(1)
+ now = time.time()
+ now = int(time.time())
+ while now >= int(time.time()):
+ time.sleep(0.1)
self._storage.pack(time.time())
# Make sure the revisions are gone, but that object zero, object2, and
# revision 3 of object1 are still there and correct.
data, revid = self._storage.load(ZERO, '')
eq(revid, revid0)
- eq(loads(data).value, 0)
+ eq(zodb_unpickle(data).value, 0)
raises(KeyError, self._storage.loadSerial, oid1, revid1)
raises(KeyError, self._storage.loadSerial, oid1, revid2)
data = self._storage.loadSerial(oid1, revid3)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid1)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 3)
data, revid = self._storage.load(oid1, '')
eq(revid, revid3)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid1)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 3)
data, revid = self._storage.load(oid2, '')
eq(revid, revid4)
- eq(loads(data).value, 11)
+ eq(zodb_unpickle(data).value, 11)
data = self._storage.loadSerial(oid2, revid4)
- pobj = pickle.loads(data)
- eq(pobj.getoid(), oid2)
+ pobj = zodb_unpickle(data)
eq(pobj.value, 11)
def checkPackUnlinkedFromRoot(self):
=== ZODB4/src/zodb/storage/tests/readonly.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/storage/tests/readonly.py:1.5 Tue Feb 25 13:55:03 2003
+++ ZODB4/src/zodb/storage/tests/readonly.py Thu Mar 13 16:32:29 2003
@@ -52,7 +52,7 @@
self.assertRaises(ReadOnlyError, self._storage.commitVersion,
'', '', t)
self.assertRaises(ReadOnlyError, self._storage.store,
- ZERO, None, '', '', t)
+ ZERO, None, '', '', '', t)
if IUndoStorage.isImplementedBy(self._storage):
self.assertRaises(ReadOnlyError, self._storage.undo,
=== ZODB4/src/zodb/storage/tests/recovery.py 1.4 => 1.5 ===
--- ZODB4/src/zodb/storage/tests/recovery.py:1.4 Wed Feb 5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/recovery.py Thu Mar 13 16:32:29 2003
@@ -23,6 +23,7 @@
from zodb.ztransaction import Transaction
from zodb.storage.tests.iterator import IteratorDeepCompare
from zodb.storage.tests.base import MinPO, zodb_unpickle
+from zodb.serialize import findrefs
class RecoveryStorage(IteratorDeepCompare):
@@ -76,8 +77,19 @@
self.assertRaises(StopIteration, txniter.next)
self.assertEqual(data.oid, oid)
self.assertEqual(data.data, None)
+ siter.close()
def checkRecoverUndoInVersion(self):
+
+ def checkObj(oid, version, v_data, nv_data):
+ # Cause test to fail if object oid has wrong data.
+ # Last two args are MinPO value on version and non-version.
+ data, revid = self._storage.load(oid, "")
+ self.assertEqual(zodb_unpickle(data).value, nv_data)
+ data, revid = self._storage.load(oid, version)
+ self.assertEqual(zodb_unpickle(data).value, v_data)
+
+
oid = self._storage.newObjectId()
version = "aVersion"
revid_a = self._dostore(oid, data=MinPO(91))
@@ -103,11 +115,7 @@
self._undo(self._storage.undoInfo()[0]['id'], oid)
self.assert_(not self._storage.versionEmpty(version))
- # check the data is what we expect it to be
- data, revid = self._storage.load(oid, version)
- self.assertEqual(zodb_unpickle(data), MinPO(92))
- data, revid = self._storage.load(oid, '')
- self.assertEqual(zodb_unpickle(data), MinPO(91))
+ checkObj(oid, version, 92, 91)
# and swap the storages
tmp = self._storage
@@ -117,11 +125,7 @@
self._undo(self._storage.undoInfo()[0]['id'], oid)
self.assert_(not self._storage.versionEmpty(version))
- # check the data is what we expect it to be
- data, revid = self._storage.load(oid, version)
- self.assertEqual(zodb_unpickle(data), MinPO(92))
- data, revid = self._storage.load(oid, '')
- self.assertEqual(zodb_unpickle(data), MinPO(91))
+ checkObj(oid, version, 92, 91)
# swap them back
self._storage = tmp
@@ -135,6 +139,7 @@
self.compare(self._storage, self._dst)
def checkRestoreAcrossPack(self):
+ eq = self.assertEqual
db = DB(self._storage)
c = db.open()
r = c.root()
@@ -150,10 +155,46 @@
# copy the final transaction manually. even though there
# was a pack, the restore() ought to succeed.
- final = list(self._storage.iterator())[-1]
+ iter = self._storage.iterator()
+ final = list(iter)[-1]
self._dst.tpcBegin(final, final.tid, final.status)
+ datarefs = []
for r in final:
- self._dst.restore(r.oid, r.serial, r.data, r.version, r.data_txn,
- final)
+ datarefs.append((r.data, r.refs))
+ self._dst.restore(r.oid, r.serial, r.data, r.refs, r.version,
+ r.data_txn, final)
self._dst.tpcVote(final)
self._dst.tpcFinish(final)
+ iter.close()
+ data, refs = datarefs[0]
+ eq(findrefs(data), refs)
+ data, refs = datarefs[1]
+ eq(data, None)
+ eq(refs, None)
+
+ def checkPackWithGCOnDestinationAfterRestore(self):
+ raises = self.assertRaises
+ db = DB(self._storage)
+ conn = db.open()
+ root = conn.root()
+ root.obj = obj1 = MinPO(1)
+ txn = get_transaction()
+ txn.note('root -> obj')
+ txn.commit()
+ root.obj.obj = obj2 = MinPO(2)
+ txn = get_transaction()
+ txn.note('root -> obj -> obj')
+ txn.commit()
+ del root.obj
+ txn = get_transaction()
+ txn.note('root -X->')
+ txn.commit()
+ # Now copy the transactions to the destination
+ self._dst.copyTransactionsFrom(self._storage)
+ # Now pack the destination
+ self._dst.pack(time.time())
+ # And check to see that the root object exists, but not the other
+ # objects.
+ data, serial = self._dst.load(root._p_oid, '')
+ raises(KeyError, self._dst.load, obj1._p_oid, '')
+ raises(KeyError, self._dst.load, obj2._p_oid, '')
=== ZODB4/src/zodb/storage/tests/revision.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/storage/tests/revision.py:1.5 Tue Feb 25 13:55:03 2003
+++ ZODB4/src/zodb/storage/tests/revision.py Thu Mar 13 16:32:29 2003
@@ -17,7 +17,7 @@
from zodb.storage.base import ZERO
from zodb.storage.tests.minpo import MinPO
-from zodb.storage.tests.base import zodb_unpickle, zodb_pickle
+from zodb.storage.tests.base import zodb_unpickle
class RevisionStorage:
=== ZODB4/src/zodb/storage/tests/synchronization.py 1.4 => 1.5 ===
--- ZODB4/src/zodb/storage/tests/synchronization.py:1.4 Wed Feb 5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/synchronization.py Thu Mar 13 16:32:29 2003
@@ -103,11 +103,11 @@
def checkStoreNotCommitting(self):
self.verifyNotCommitting(self._storage.store,
- OID, SERIALNO, "", "", Transaction())
+ OID, SERIALNO, "", "", "", Transaction())
def checkStoreWrongTrans(self):
self.verifyWrongTrans(self._storage.store,
- OID, SERIALNO, "", "", Transaction())
+ OID, SERIALNO, "", "", "", Transaction())
## def checkNewOidNotCommitting(self):
## self.verifyNotCommitting(self._storage.new_oid)
=== ZODB4/src/zodb/storage/tests/test_autopack.py 1.9 => 1.10 ===
--- ZODB4/src/zodb/storage/tests/test_autopack.py:1.9 Wed Feb 5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/test_autopack.py Thu Mar 13 16:32:29 2003
@@ -287,7 +287,8 @@
obj2 = C()
t = Transaction()
storage.tpcBegin(t)
- obj2sn = storage.store('\0'*7 + '\2', ZERO, zodb_pickle(obj2), '', t)
+ data, refs = zodb_pickle(obj2)
+ obj2sn = storage.store('\0'*7 + '\2', ZERO, data, refs, '', t)
# Now, acquire the condvar lock and start a thread that will do a
# pack, up to the _sweep call. Wait for the _mark() call to
# complete.
@@ -302,7 +303,8 @@
# links the object to root.
root.obj2 = obj2
rootsn = storage.getSerial(ZERO)
- rootsn = storage.store(ZERO, rootsn, zodb_pickle(root), '', t)
+ data, refs = zodb_pickle(root)
+ rootsn = storage.store(ZERO, rootsn, data, refs, '', t)
storage.tpcVote(t)
storage.tpcFinish(t)
# And notify the pack thread that it can do the sweep and collect
@@ -349,12 +351,14 @@
packthread = self._getPackThread(storage)
packthread.start()
self._cv.wait()
- obj2sn = storage.store('\0'*7 + '\2', ZERO, zodb_pickle(obj2), '', t)
+ data, refs = zodb_pickle(obj2)
+ obj2sn = storage.store('\0'*7 + '\2', ZERO, data, refs, '', t)
# Now that the _mark() has finished, complete the transaction, which
# links the object to root.
root.obj2 = obj2
rootsn = storage.getSerial(ZERO)
- rootsn = storage.store(ZERO, rootsn, zodb_pickle(root), '', t)
+ data, refs = zodb_pickle(root)
+ rootsn = storage.store(ZERO, rootsn, data, refs, '', t)
storage.tpcVote(t)
storage.tpcFinish(t)
# And notify the pack thread that it can do the sweep and collect
=== ZODB4/src/zodb/storage/tests/test_file.py 1.6 => 1.7 ===
--- ZODB4/src/zodb/storage/tests/test_file.py:1.6 Wed Feb 5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/test_file.py Thu Mar 13 16:32:29 2003
@@ -11,9 +11,12 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-import zodb.storage.file
-import sys, os, unittest
+import os
+import sys
import errno
+import unittest
+
+import zodb.storage.file
from zodb.ztransaction import Transaction
from zodb.storage.interfaces import *
@@ -23,6 +26,8 @@
from zodb.storage.tests.base import MinPO, zodb_unpickle
+from transaction import get_transaction
+
class FileStorageTests(
base.StorageTestBase,
basic.BasicStorage,
@@ -50,6 +55,7 @@
self.open(create=1)
def tearDown(self):
+ get_transaction().abort()
self._test_read_index()
self._storage.close()
base.removefs("FileStorageTests.fs")
@@ -65,7 +71,7 @@
from zodb.storage.fsdump import Dumper
Dumper("FileStorageTests.fs").dump()
raise
-
+
L1 = copy._index.items()
L2 = self._storage._index.items()
L1.sort(); L2.sort()
=== ZODB4/src/zodb/storage/tests/undo.py 1.6 => 1.7 ===
--- ZODB4/src/zodb/storage/tests/undo.py:1.6 Fri Feb 28 11:16:57 2003
+++ ZODB4/src/zodb/storage/tests/undo.py Thu Mar 13 16:32:29 2003
@@ -4,11 +4,11 @@
"""
import time
-import types
from zodb import interfaces
+from zodb.interfaces import ZERO
from zodb.ztransaction import Transaction
-from zodb.utils import u64, p64, z64
+from zodb.utils import u64, p64
from zodb.db import DB
from zodb.storage.tests.minpo import MinPO
@@ -25,10 +25,10 @@
def _transaction_begin(self):
self.__serials = {}
- def _transaction_store(self, oid, rev, data, vers, trans):
- r = self._storage.store(oid, rev, data, vers, trans)
+ def _transaction_store(self, oid, rev, (data, refs), vers, trans):
+ r = self._storage.store(oid, rev, data, refs, vers, trans)
if r:
- if type(r) == types.StringType:
+ if isinstance(r, str):
self.__serials[oid] = r
else:
for oid, serial in r:
@@ -43,13 +43,16 @@
def _transaction_newserial(self, oid):
return self.__serials[oid]
- def _multi_obj_transaction(self, objs):
+ def _multi_obj_transaction(self, objs, note=None):
newrevs = {}
t = Transaction()
+ if note:
+ t.note(note)
self._storage.tpcBegin(t)
self._transaction_begin()
- for oid, rev, data in objs:
- self._transaction_store(oid, rev, data, '', t)
+ # datarefs is a tuple of (data, refs)
+ for oid, rev, datarefs in objs:
+ self._transaction_store(oid, rev, datarefs, '', t)
newrevs[oid] = None
self._transaction_vote(t)
self._storage.tpcFinish(t)
@@ -196,11 +199,10 @@
def checkTwoObjectUndo(self):
eq = self.assertEqual
# Convenience
- p31, p32, p51, p52 = map(zodb_pickle,
- map(MinPO, (31, 32, 51, 52)))
+ p31, p32, p51, p52 = [zodb_pickle(MinPO(i)) for i in (31, 32, 51, 52)]
oid1 = self._storage.newObjectId()
oid2 = self._storage.newObjectId()
- revid1 = revid2 = z64
+ revid1 = revid2 = ZERO
# Store two objects in the same transaction
t = Transaction()
self._storage.tpcBegin(t)
@@ -251,26 +253,25 @@
# Convenience
eq = self.assertEqual
unless = self.failUnless
- p30, p31, p32, p50, p51, p52 = map(zodb_pickle,
- map(MinPO,
- (30, 31, 32, 50, 51, 52)))
+ p30, p31, p32, p50, p51, p52 = [zodb_pickle(MinPO(i))
+ for i in (30, 31, 32, 50, 51, 52)]
oid1 = self._storage.newObjectId()
oid2 = self._storage.newObjectId()
- revid1 = revid2 = z64
+ revid1 = revid2 = ZERO
# Store two objects in the same transaction
d = self._multi_obj_transaction([(oid1, revid1, p30),
(oid2, revid2, p50),
- ])
+ ], "first update")
eq(d[oid1], d[oid2])
# Update those same two objects
d = self._multi_obj_transaction([(oid1, d[oid1], p31),
(oid2, d[oid2], p51),
- ])
+ ], "second update")
eq(d[oid1], d[oid2])
# Update those same two objects
d = self._multi_obj_transaction([(oid1, d[oid1], p32),
(oid2, d[oid2], p52),
- ])
+ ], "third update")
eq(d[oid1], d[oid2])
revid1 = self._transaction_newserial(oid1)
revid2 = self._transaction_newserial(oid2)
@@ -285,14 +286,12 @@
tid = info[0]['id']
tid1 = info[1]['id']
t = Transaction()
+ t.note("undo transaction containing two objects")
self._storage.tpcBegin(t)
oids = self._storage.undo(tid, t)
oids1 = self._storage.undo(tid1, t)
self._storage.tpcVote(t)
self._storage.tpcFinish(t)
- # We get the finalization stuff called an extra time:
-## self._storage.tpc_vote(t)
-## self._storage.tpc_finish(t)
eq(len(oids), 2)
eq(len(oids1), 2)
unless(oid1 in oids)
@@ -305,6 +304,7 @@
info = self._storage.undoInfo()
tid = info[0]['id']
t = Transaction()
+ t.note("undo the undo")
self._storage.tpcBegin(t)
oids = self._storage.undo(tid, t)
self._storage.tpcVote(t)
@@ -320,15 +320,14 @@
def checkTwoObjectUndoAgain(self):
eq = self.assertEqual
- p31, p32, p33, p51, p52, p53 = map(
- zodb_pickle,
- map(MinPO, (31, 32, 33, 51, 52, 53)))
+ p31, p32, p33, p51, p52, p53 = [zodb_pickle(MinPO(i))
+ for i in (31, 32, 33, 51, 52, 53)]
# Like the above, but the first revision of the objects are stored in
# different transactions.
oid1 = self._storage.newObjectId()
oid2 = self._storage.newObjectId()
- revid1 = self._dostore(oid1, data=p31, already_pickled=1)
- revid2 = self._dostore(oid2, data=p51, already_pickled=1)
+ revid1 = self._dostore(oid1, data=p31, already_pickled=True)
+ revid2 = self._dostore(oid2, data=p51, already_pickled=True)
# Update those same two objects
t = Transaction()
self._storage.tpcBegin(t)
@@ -412,10 +411,8 @@
oid1 = oid
revid1 = revid_c
oid2 = self._storage.newObjectId()
- revid2 = z64
- p81, p82, p91, p92 = map(zodb_pickle,
- map(MinPO, (81, 82, 91, 92)))
-
+ revid2 = ZERO
+ p81, p82, p91, p92 = [zodb_pickle(MinPO(i)) for i in (81, 82, 91, 92)]
t = Transaction()
self._storage.tpcBegin(t)
self._transaction_begin()
@@ -450,8 +447,10 @@
def checkTransactionalUndoAfterPack(self):
eq = self.assertEqual
# Add a few object revisions
- oid = self._storage.newObjectId()
- revid1 = self._dostore(oid, data=MinPO(51))
+ self._initroot()
+ obj, root, revid0 = self._linked_newobj()
+ oid = obj._p_oid
+ revid1 = self._dostore(oid, data=MinPO(51), revid=obj._p_serial)
# For the pack(), we need a timestamp greater than revid1's timestamp.
# The semantics of pack()'s `t' argument is that all non-current
# revisions with an earlier timestamp will be packed away. If they
@@ -468,12 +467,14 @@
revid3 = self._dostore(oid, revid=revid2, data=MinPO(53))
# Now get the undo log
info = self._storage.undoInfo()
- eq(len(info), 3)
+ eq(len(info), 5)
tid = info[0]['id']
+
# Now pack just the initial revision of the object. We need the
# second revision otherwise we won't be able to undo the third
# revision!
self._storage.pack(packtime)
+
# Make some basic assertions about the undo information now
info2 = self._storage.undoInfo()
eq(len(info2), 2)
@@ -564,7 +565,8 @@
for j in range(OBJECTS):
oid = s.newObjectId()
obj = MinPO(i * OBJECTS + j)
- revid = s.store(oid, None, zodb_pickle(obj), '', t)
+ data, refs = zodb_pickle(obj)
+ revid = s.store(oid, None, data, refs, '', t)
orig.append((tid, oid, revid))
s.tpcVote(t)
s.tpcFinish(t)
=== ZODB4/src/zodb/storage/tests/undoversion.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/storage/tests/undoversion.py:1.3 Wed Feb 5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/undoversion.py Thu Mar 13 16:32:29 2003
@@ -28,13 +28,16 @@
version=version)
revid_c = self._dostore(oid, revid=revid_b, data=MinPO(93),
version=version)
+
info=self._storage.undoInfo()
tid=info[0]['id']
t = Transaction()
+ t.note("undo last revision (93)")
self._storage.tpcBegin(t)
oids = self._storage.undo(tid, t)
self._storage.tpcVote(t)
self._storage.tpcFinish(t)
+
assert len(oids) == 1
assert oids[0] == oid
data, revid = self._storage.load(oid, '')
@@ -45,10 +48,12 @@
assert zodb_unpickle(data) == MinPO(92)
# Now commit the version...
t = Transaction()
+ t.note("commit version")
self._storage.tpcBegin(t)
oids = self._storage.commitVersion(version, '', t)
self._storage.tpcVote(t)
self._storage.tpcFinish(t)
+
assert len(oids) == 1
assert oids[0] == oid
@@ -60,16 +65,18 @@
info=self._storage.undoInfo()
tid=info[0]['id']
t = Transaction()
+ t.note("undo commit version")
self._storage.tpcBegin(t)
oids = self._storage.undo(tid, t)
self._storage.tpcVote(t)
self._storage.tpcFinish(t)
+
assert len(oids) == 1
assert oids[0] == oid
data, revid = self._storage.load(oid, version)
- assert zodb_unpickle(data) == MinPO(92)
+ self.assertEqual(zodb_unpickle(data), MinPO(92))
data, revid = self._storage.load(oid, '')
- assert zodb_unpickle(data) == MinPO(91)
+ self.assertEqual(zodb_unpickle(data), MinPO(91))
# Now abort the version
t = Transaction()
self._storage.tpcBegin(t)
=== ZODB4/src/zodb/storage/tests/version.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/storage/tests/version.py:1.3 Wed Feb 5 18:28:27 2003
+++ ZODB4/src/zodb/storage/tests/version.py Thu Mar 13 16:32:29 2003
@@ -16,18 +16,23 @@
Any storage that supports versions should be able to pass all these tests.
"""
+import time
+
from zodb import interfaces
+from zodb.db import DB
from zodb.ztransaction import Transaction
from zodb.storage.tests.minpo import MinPO
from zodb.storage.tests.base import zodb_unpickle
from zodb.storage.interfaces import IUndoStorage, IVersionStorage
+from transaction import get_transaction
+
class VersionStorage:
def checkIVersionStorage(self):
self.assert_(IVersionStorage.isImplementedBy(self._storage))
-
+
def checkVersionedStoreAndLoad(self):
eq = self.assertEqual
# Store a couple of non-version revisions of the object
@@ -117,10 +122,6 @@
unless('one' in versions)
unless('two' in versions)
unless('three' in versions)
- # Now flex the `max' argument
- versions = self._storage.versions(1)
- self.assertEqual(len(versions), 1)
- unless('one' in versions or 'two' in versions or 'three' in versions)
def _setup_version(self, version='test-version'):
# Store some revisions in the non-version
@@ -332,3 +333,45 @@
self._storage.load, oid, '')
self.assertRaises(KeyError,
self._storage.load, oid, 'two')
+
+ def checkPackVersions(self):
+ db = DB(self._storage)
+ cn = db.open(version="testversion")
+ root = cn.root()
+
+ obj = root["obj"] = MinPO("obj")
+ root["obj2"] = MinPO("obj2")
+ txn = get_transaction()
+ txn.note("create 2 objs in version")
+ txn.commit()
+
+ obj.value = "77"
+ txn = get_transaction()
+ txn.note("modify obj in version")
+ txn.commit()
+
+ # undo the modification to generate a mix of backpointers
+ # and versions for pack to chase
+ info = db.undoInfo()
+ db.undo(info[0]["id"])
+ txn = get_transaction()
+ txn.note("undo modification")
+ txn.commit()
+
+ self._storage.pack(time.time())
+
+ db.commitVersion("testversion")
+ txn = get_transaction()
+ txn.note("commit version")
+ txn.commit()
+
+ cn = db.open()
+ root = cn.root()
+ root["obj"] = "no version"
+
+ txn = get_transaction()
+ txn.note("modify obj")
+ txn.commit()
+
+ self._storage.pack(time.time())
+