[Zodb-checkins] SVN: ZODB/branches/gocept-iteration/src/Z - Last
snapshot for today.
Christian Theune
ct at gocept.com
Thu Feb 14 11:27:37 EST 2008
Log message for revision 83838:
- Last snapshot for today.
- We cleaned up a bit and made sure that iterators behave equally in that they
can only be iterated over once.
- Fixed some bugs in the various iterator implementations and embedded
iterator test base classes in more places.
Changed:
U ZODB/branches/gocept-iteration/src/ZEO/ClientStorage.py
U ZODB/branches/gocept-iteration/src/ZEO/StorageServer.py
U ZODB/branches/gocept-iteration/src/ZEO/tests/testZEO.py
U ZODB/branches/gocept-iteration/src/ZODB/BaseStorage.py
U ZODB/branches/gocept-iteration/src/ZODB/DemoStorage.py
U ZODB/branches/gocept-iteration/src/ZODB/FileStorage/FileStorage.py
U ZODB/branches/gocept-iteration/src/ZODB/FileStorage/__init__.py
U ZODB/branches/gocept-iteration/src/ZODB/MappingStorage.py
U ZODB/branches/gocept-iteration/src/ZODB/fsrecover.py
U ZODB/branches/gocept-iteration/src/ZODB/interfaces.py
U ZODB/branches/gocept-iteration/src/ZODB/tests/IteratorStorage.py
U ZODB/branches/gocept-iteration/src/ZODB/tests/TransactionalUndoStorage.py
U ZODB/branches/gocept-iteration/src/ZODB/tests/testDemoStorage.py
U ZODB/branches/gocept-iteration/src/ZODB/tests/testMappingStorage.py
-=-
Modified: ZODB/branches/gocept-iteration/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZEO/ClientStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZEO/ClientStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -1265,10 +1265,14 @@
# disposed it.
self._forget_iterator(iid)
break
- yield ClientStorageTransactionInformation(self, *item)
- def _setup_iterator(self, factory, iid):
- self._iterators[iid] = iterator = factory(iid)
+ tid = item[0]
+ riid = self._server.iterator_record_start(tid)
+ yield self._setup_iterator(ClientStorageTransactionInformation,
+ riid, self, *item)
+
+ def _setup_iterator(self, factory, iid, *args):
+ self._iterators[iid] = iterator = factory(iid, *args)
self._iterator_ids.add(iid)
return iterator
@@ -1293,8 +1297,10 @@
class ClientStorageTransactionInformation(ZODB.BaseStorage.TransactionRecord):
- def __init__(self, storage, tid, status, user, description, extension):
+ def __init__(self, riid, storage, tid, status, user, description, extension):
self._storage = storage
+ self._riid = riid
+ self._completed = False
self.tid = tid
self.status = status
@@ -1303,15 +1309,18 @@
self.extension = extension
def __iter__(self):
- riid = self._storage._server.iterator_record_start(self.tid)
- return self._storage._setup_iterator(self._iterator, riid)
+ return self
- def _iterator(self, riid):
- while True:
- item = self._storage._server.iterator_record_next(riid)
- if item is None:
- # The iterator is exhausted, and the server has already
- # disposed it.
- self._storage._forget_iterator(riid)
- break
- yield ZODB.BaseStorage.DataRecord(*item)
+ def next(self):
+ if self._completed:
+ # We finished iteration once already and the server can't know
+ # about the iteration anymore.
+ raise StopIteration
+ item = self._storage._server.iterator_record_next(self._riid)
+ if item is None:
+ # The iterator is exhausted, and the server has already
+ # disposed it.
+ self._storage._forget_iterator(self._riid)
+ self._completed = True
+ raise StopIteration
+ return ZODB.BaseStorage.DataRecord(*item)
Modified: ZODB/branches/gocept-iteration/src/ZEO/StorageServer.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZEO/StorageServer.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZEO/StorageServer.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -689,7 +689,7 @@
def iterator_start(self, start, stop):
iid = self._iterator_ids.next()
- self._iterators[iid] = self.storage.iterator(start, stop)
+ self._iterators[iid] = iter(self.storage.iterator(start, stop))
return iid
def iterator_next(self, iid):
@@ -710,7 +710,7 @@
def iterator_record_start(self, tid):
iid = self._iterator_ids.next()
txn_infos = list(self.storage.iterator(tid, tid))
- assert len(txn_infos) == 1
+ assert len(txn_infos) == 1, "%s" % txn_infos
self._iterators[iid] = iter(txn_infos[0])
return iid
Modified: ZODB/branches/gocept-iteration/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZEO/tests/testZEO.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZEO/tests/testZEO.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -272,6 +272,16 @@
def getConfig(self):
return """<mappingstorage 1/>"""
+ def checkSimpleIteration(self):
+ # The test base class IteratorStorage assumes that we keep undo data
+ # to construct our iterator, which we don't, so we disable this test.
+ pass
+
+ def checkUndoZombie(self):
+ # The test base class IteratorStorage assumes that we keep undo data
+ # to construct our iterator, which we don't, so we disable this test.
+ pass
+
class DemoStorageTests(
GenericTests,
):
@@ -285,6 +295,11 @@
</demostorage>
""" % tempfile.mktemp()
+ def checkUndoZombie(self):
+ # The test base class IteratorStorage assumes that we keep undo data
+ # to construct our iterator, which we don't, so we disable this test.
+ pass
+
class HeartbeatTests(ZEO.tests.ConnectionTests.CommonSetupTearDown):
"""Make sure a heartbeat is being sent and that it does no harm
Modified: ZODB/branches/gocept-iteration/src/ZODB/BaseStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/BaseStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/BaseStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -364,7 +364,15 @@
self.description = description
self.extension = extension
+ # XXX This is a workaround to make the TransactionRecord compatible with a
+ # transaction object because it is passe
+ def _ext_set(self, value):
+ self.extension = value
+ def _ext_get(self):
+ return self.extension
+ _extension = property(fset=_ext_set, fget=_ext_get)
+
class DataRecord(object):
"""Abstract base class for iterator protocol"""
Modified: ZODB/branches/gocept-iteration/src/ZODB/DemoStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/DemoStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/DemoStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -576,7 +576,7 @@
if self._base is not None:
self._base.close()
- def iterator(self, start, stop):
+ def iterator(self, start=None, stop=None):
for tid, (packed, user, description, extension, records) \
in self._data.items():
if tid < start:
@@ -597,14 +597,17 @@
def __init__(self, tid, status, user, description, extension, records):
super(TransactionRecord, self).__init__(
tid, status, user, description, extension)
- self._records = records
+ self._records = list(records)
def __iter__(self):
- for oid, pre, vdata, data, tid in self._records:
+ while self._records:
+ oid, prev, vdata, data, tid = self._records.pop()
if vdata is None:
version = ''
else:
version, data = vdata
- prev = pre[-1] # pre is supposed to be the previous data record,
- # which has its tid as its last element
+ if prev is not None:
+ # prev is supposed to be the previous data record,
+ # which has its tid as its last element
+ prev = prev[-1]
yield ZODB.BaseStorage.DataRecord(oid, tid, data, version, prev)
Modified: ZODB/branches/gocept-iteration/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/FileStorage/FileStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/FileStorage/FileStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -1553,9 +1553,6 @@
def iterator(self):
return self
- def __iter__(self):
- return self
-
def close(self):
file = self._file
if file is not None:
@@ -1590,14 +1587,19 @@
panic("%s has inconsistent transaction length at %s "
"(%s != %s)", file.name, pos, u64(rtl), u64(stl))
+ # Iterator protocol
+ def __iter__(self):
+ return self
+
def next(self):
if self._file is None:
# A closed iterator. Is IOError the best we can do? For
# now, mimic a read on a closed file.
- raise IOError('iterator is closed')
+ raise IOError("The iterator's file is closed.")
pos = self._pos
- while 1:
+ while True:
+
# Read the transaction record
try:
h = self._read_txn_header(pos)
@@ -1613,11 +1615,11 @@
self._ltid = h.tid
if self._stop is not None and h.tid > self._stop:
- raise StopIteration
+ break
if h.status == "c":
# Assume we've hit the last, in-progress transaction
- raise StopIteration
+ break
if pos + h.tlen + 8 > self._file_size:
# Hm, the data were truncated or the checkpoint flag wasn't
@@ -1667,8 +1669,8 @@
except:
pass
- result = RecordIterator(h.tid, h.status, h.user, h.descr,
- e, pos, tend, self._file, tpos)
+ result = TransactionRecord(h.tid, h.status, h.user, h.descr,
+ e, pos, tend, self._file, tpos)
# Read the (intentionally redundant) transaction length
self._file.seek(tend)
@@ -1684,7 +1686,7 @@
raise StopIteration
-class RecordIterator(BaseStorage.TransactionRecord, FileStorageFormatter):
+class TransactionRecord(BaseStorage.TransactionRecord, FileStorageFormatter):
"""Iterate over the transactions in a FileStorage file."""
def __init__(self, tid, status, user, desc, ext, pos, tend, file, tpos):
@@ -1727,8 +1729,7 @@
# Should it go to the original data like BDBFullStorage?
prev_txn = self.getTxnFromData(h.oid, h.back)
- r = Record(h.oid, h.tid, data, prev_txn, pos)
- return r
+ return Record(h.oid, h.tid, data, prev_txn, pos)
raise StopIteration
Modified: ZODB/branches/gocept-iteration/src/ZODB/FileStorage/__init__.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/FileStorage/__init__.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/FileStorage/__init__.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -1,4 +1,4 @@
# this is a package
-from ZODB.FileStorage.FileStorage import FileStorage, RecordIterator
+from ZODB.FileStorage.FileStorage import FileStorage, TransactionRecord
from ZODB.FileStorage.FileStorage import FileIterator, Record, packed_version
Modified: ZODB/branches/gocept-iteration/src/ZODB/MappingStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/MappingStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/MappingStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -148,8 +148,11 @@
tid = odata[:8]
oids = tid2oid.setdefault(tid, [])
oids.append(oid)
-
for tid, oids in sorted(tid2oid.items()):
+ if tid < start:
+ continue
+ if stop is not None and tid > stop:
+ break
yield TransactionRecord(self, tid, oids)
@@ -158,10 +161,15 @@
def __init__(self, storage, tid, oids):
super(TransactionRecord, self).__init__(tid, 'p', '', '', {})
self._storage = storage
- self._oids = oids
+ self._oids = list(oids)
def __iter__(self):
- for oid in self._oids:
+ return self
+
+ def next(self):
+ while self._oids:
+ oid = self._oids.pop()
storage_data = self._storage._index[oid]
tid, data = storage_data[:8], storage_data[8:]
- yield ZODB.BaseStorage.DataRecord(oid, tid, data, '', None)
+ return ZODB.BaseStorage.DataRecord(oid, tid, data, '', None)
+ raise StopIteration
Modified: ZODB/branches/gocept-iteration/src/ZODB/fsrecover.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/fsrecover.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/fsrecover.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -82,7 +82,7 @@
import ZODB.FileStorage
from ZODB.utils import u64
-from ZODB.FileStorage import RecordIterator
+from ZODB.FileStorage import TransactionRecord
from persistent.TimeStamp import TimeStamp
@@ -146,8 +146,8 @@
except: e={}
else: e={}
- result = RecordIterator(tid, status, user, description, e, pos, tend,
- f, tpos)
+ result = TransactionRecord(tid, status, user, description, e, pos, tend,
+ f, tpos)
pos = tend
# Read the (intentionally redundant) transaction length
Modified: ZODB/branches/gocept-iteration/src/ZODB/interfaces.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/interfaces.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/interfaces.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -787,7 +787,10 @@
class IStorageTransactionInformation(Interface):
- """Provide information about a storage transaction
+ """Provide information about a storage transaction.
+
+ Can be iterated over to retrieve the records modified in the transaction.
+
"""
tid = Attribute("Transaction id")
@@ -797,9 +800,12 @@
extension = Attribute("A dictionary carrying the transaction's extension data")
def __iter__():
- """Return an iterable of IStorageRecordInformation
+ """Iterate over the transaction's records given as
+ IStorageRecordInformation objects.
+
"""
+
class IStorageIteration(Interface):
"""API for iterating over the contents of a storage
@@ -811,9 +817,6 @@
def iterator(start=None, stop=None):
"""Return an IStorageTransactionInformation iterator.
- An IStorageTransactionInformation iterator is returned for
- iterating over the transactions in the storage.
-
If the start argument is not None, then iteration will start
with the first transaction whose identifier is greater than or
equal to start.
@@ -824,6 +827,7 @@
"""
+
class IStorageUndoable(IStorage):
"""A storage supporting transactional undo.
"""
Modified: ZODB/branches/gocept-iteration/src/ZODB/tests/IteratorStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/tests/IteratorStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/tests/IteratorStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -37,7 +37,9 @@
eq(zodb_unpickle(rec.data), MinPO(val))
val = val + 1
eq(val, val0 + len(revids))
- txniter.close()
+ if hasattr(txniter, 'close'):
+ # XXX See bug #191573
+ txniter.close()
class IteratorStorage(IteratorCompare):
@@ -55,8 +57,10 @@
self._oid = oid = self._storage.new_oid()
revid1 = self._dostore(oid, data=MinPO(11))
txniter = self._storage.iterator()
- txniter.close()
- self.assertRaises(IOError, txniter.next)
+ if hasattr(txniter, 'close'):
+ # XXX See bug #191573
+ txniter.close()
+ self.assertRaises(IOError, txniter.next)
def checkUndoZombie(self):
oid = self._storage.new_oid()
@@ -129,9 +133,22 @@
match = True
if not match:
self.fail("Could not find transaction with matching id")
-
+ def checkIterateRepeatedly(self):
+ self._dostore()
+ transactions = self._storage.iterator()
+ self.assertEquals(1, len(list(transactions)))
+ # The iterator can only be consumed once:
+ self.assertEquals(0, len(list(transactions)))
+ def checkIterateRecordsRepeatedly(self):
+ self._dostore()
+ tinfo = self._storage.iterator().next()
+ self.assertEquals(1, len(list(tinfo)))
+ # The iterator can only be consumed once:
+ self.assertEquals(0, len(list(tinfo)))
+
+
class ExtendedIteratorStorage(IteratorCompare):
def checkExtendedIteration(self):
Modified: ZODB/branches/gocept-iteration/src/ZODB/tests/TransactionalUndoStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/tests/TransactionalUndoStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/tests/TransactionalUndoStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -670,11 +670,11 @@
# OBJECTS * BATCHES modifications, followed by
# BATCHES undos
- iter = s.iterator()
+ transactions = s.iterator()
eq = self.assertEqual
for i in range(BATCHES):
- txn = iter.next()
+ txn = transactions.next()
tid = p64(i + 1)
eq(txn.tid, tid)
@@ -686,11 +686,11 @@
eq(L1, L2)
for i in range(BATCHES * OBJECTS):
- txn = iter.next()
+ txn = transactions.next()
eq(len([rec for rec in txn if rec.data_txn is None]), 1)
for i in range(BATCHES):
- txn = iter.next()
+ txn = transactions.next()
# The undos are performed in reverse order.
otid = p64(BATCHES - i)
@@ -701,7 +701,7 @@
L2.sort()
eq(L1, L2)
- self.assertRaises(StopIteration, iter.next)
+ self.assertRaises(StopIteration, transactions.next)
def checkUndoLogMetadata(self):
# test that the metadata is correct in the undo log
Modified: ZODB/branches/gocept-iteration/src/ZODB/tests/testDemoStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/tests/testDemoStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/tests/testDemoStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -18,11 +18,12 @@
import ZODB.utils
import ZODB.DemoStorage
from ZODB.tests import StorageTestBase, BasicStorage
-from ZODB.tests import Synchronization
+from ZODB.tests import Synchronization, IteratorStorage
class DemoStorageTests(StorageTestBase.StorageTestBase,
BasicStorage.BasicStorage,
Synchronization.SynchronizedStorage,
+ IteratorStorage.IteratorStorage
):
def setUp(self):
@@ -44,6 +45,10 @@
self.assertEqual(s2.load(ZODB.utils.z64, ''),
self._storage.load(ZODB.utils.z64, ''))
+ def checkUndoZombie(self):
+ # The test base class IteratorStorage assumes that we keep undo data
+ # to construct our iterator, which we don't, so we disable this test.
+ pass
class DemoStorageWrappedBase(DemoStorageTests):
Modified: ZODB/branches/gocept-iteration/src/ZODB/tests/testMappingStorage.py
===================================================================
--- ZODB/branches/gocept-iteration/src/ZODB/tests/testMappingStorage.py 2008-02-14 15:24:04 UTC (rev 83837)
+++ ZODB/branches/gocept-iteration/src/ZODB/tests/testMappingStorage.py 2008-02-14 16:27:36 UTC (rev 83838)
@@ -16,13 +16,14 @@
from ZODB.tests import StorageTestBase
from ZODB.tests import BasicStorage, MTStorage, Synchronization
-from ZODB.tests import PackableStorage
+from ZODB.tests import PackableStorage, IteratorStorage
class MappingStorageTests(StorageTestBase.StorageTestBase,
BasicStorage.BasicStorage,
MTStorage.MTStorage,
PackableStorage.PackableStorage,
Synchronization.SynchronizedStorage,
+ IteratorStorage.IteratorStorage
):
def setUp(self):
@@ -37,6 +38,16 @@
# have this limit, so we inhibit this test here.
pass
+ def checkSimpleIteration(self):
+ # The test base class IteratorStorage assumes that we keep undo data
+ # to construct our iterator, which we don't, so we disable this test.
+ pass
+
+ def checkUndoZombie(self):
+ # The test base class IteratorStorage assumes that we keep undo data
+ # to construct our iterator, which we don't, so we disable this test.
+ pass
+
def test_suite():
suite = unittest.makeSuite(MappingStorageTests, 'check')
return suite
More information about the Zodb-checkins
mailing list