[Zodb-checkins] SVN: ZODB/trunk/src/ Merged Dieter's branch that ads an option to drop a zeo cache rather
Jim Fulton
jim at zope.com
Wed Sep 10 19:15:15 EDT 2008
Log message for revision 91047:
Merged Dieter's branch that ads an option to drop a zeo cache rather
than verifying it.
Changed:
U ZODB/trunk/src/CHANGES.txt
U ZODB/trunk/src/ZEO/ClientStorage.py
U ZODB/trunk/src/ZEO/component.xml
U ZODB/trunk/src/ZEO/tests/testZEO.py
U ZODB/trunk/src/ZODB/component.xml
U ZODB/trunk/src/ZODB/config.py
-=-
Modified: ZODB/trunk/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt 2008-09-10 22:13:32 UTC (rev 91046)
+++ ZODB/trunk/src/CHANGES.txt 2008-09-10 23:15:14 UTC (rev 91047)
@@ -8,6 +8,12 @@
New Features
------------
+- New `ClientStorage` configuration option `drop_cache_rather_verify`.
+ If this option is true then the ZEO client cache is dropped instead of
+ the long (unoptimized) verification. For large caches, setting this
+ option can avoid effective downtimes in the order of hours when
+ the connection to the ZEO server was interrupted for a longer time.
+
- The connection now estimates the object size based on its pickle size
and informs the cache about size changes.
Modified: ZODB/trunk/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/trunk/src/ZEO/ClientStorage.py 2008-09-10 22:13:32 UTC (rev 91046)
+++ ZODB/trunk/src/ZEO/ClientStorage.py 2008-09-10 23:15:14 UTC (rev 91047)
@@ -122,6 +122,7 @@
wait_for_server_on_startup=None, # deprecated alias for wait
wait=None, wait_timeout=None,
read_only=0, read_only_fallback=0,
+ drop_cache_rather_verify=False,
username='', password='', realm=None,
blob_dir=None, shared_blob_dir=False):
"""ClientStorage constructor.
@@ -196,6 +197,9 @@
realm -- not documented.
+ drop_cache_rather_verify -- a flag indicating that the cache
+ should be dropped rather than expensively verified.
+
blob_dir -- directory path for blob data. 'blob data' is data that
is retrieved via the loadBlob API.
@@ -218,6 +222,14 @@
if debug:
log2("ClientStorage(): debug argument is no longer used")
+ # Remember some parameters for "_setupCache"
+ self._var_ = var
+ self._storage_ = storage
+ self._client_ = client
+ self._cache_size_ = cache_size
+
+ self._drop_cache_rather_verify = drop_cache_rather_verify
+
# wait defaults to True, but wait_for_server_on_startup overrides
# if not None
if wait_for_server_on_startup is not None:
@@ -342,13 +354,7 @@
else:
self.fshelper = None
- # Decide whether to use non-temporary files
- if client is not None:
- dir = var or os.getcwd()
- cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
- else:
- cache_path = None
- self._cache = self.ClientCacheClass(cache_path, size=cache_size)
+ self._setupCache()
self._rpc_mgr = self.ConnectionManagerClass(addr, self,
tmin=min_disconnect_poll,
@@ -363,6 +369,19 @@
if not self._rpc_mgr.attempt_connect():
self._rpc_mgr.connect()
+ def _setupCache(self):
+ '''create and open the cache.'''
+ # Decide whether to use non-temporary files
+ storage = self._storage_
+ client = self._client_
+ cache_size = self._cache_size_
+ if client is not None:
+ dir = self._var_ or os.getcwd()
+ cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
+ else:
+ cache_path = None
+ self._cache = self.ClientCacheClass(cache_path, size=cache_size)
+
def _wait(self, timeout=None):
if timeout is not None:
deadline = time.time() + timeout
@@ -1225,6 +1244,23 @@
elif ltid and ltid != utils.z64:
self._cache.setLastTid(ltid)
+ # From this point on, we do not have complete information about
+ # the missed transactions. The reason is that cache
+ # verification only checks objects in the client cache and
+ # there may be objects in the object caches that aren't in the
+ # client cach that would need verification too. We avoid that
+ # problem by just invalidating the objects in the object caches.
+ if self._db is not None:
+ self._db.invalidateCache()
+
+ if self._cache and self._drop_cache_rather_verify:
+ log2("dropping cache")
+ self._cache.close()
+ self._setupCache() # creates a new cache
+ self._server = server
+ self._ready.set()
+ return "cache dropped"
+
log2("Verifying cache")
for oid, tid in self._cache.contents():
server.verify(oid, tid)
@@ -1381,7 +1417,8 @@
class ClientStorageTransactionInformation(ZODB.BaseStorage.TransactionRecord):
- def __init__(self, storage, txiter, tid, status, user, description, extension):
+ def __init__(self, storage, txiter, tid, status, user, description,
+ extension):
self._storage = storage
self._txiter = txiter
self._completed = False
@@ -1394,7 +1431,8 @@
self.extension = extension
def __iter__(self):
- riid = self._storage._server.iterator_record_start(self._txiter._iid, self.tid)
+ riid = self._storage._server.iterator_record_start(self._txiter._iid,
+ self.tid)
return self._storage._setup_iterator(RecordIterator, riid)
Modified: ZODB/trunk/src/ZEO/component.xml
===================================================================
--- ZODB/trunk/src/ZEO/component.xml 2008-09-10 22:13:32 UTC (rev 91046)
+++ ZODB/trunk/src/ZEO/component.xml 2008-09-10 23:15:14 UTC (rev 91047)
@@ -102,6 +102,16 @@
<metadefault>$INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid)</metadefault>
</key>
+ <!-- DM 2006-06-12: added option -->
+ <key name="drop-cache-rather-verify" datatype="boolean"
+ required="no" default="false">
+ <description>
+ indicates that the cache should be dropped rather than
+ verified when the verification optimization is not
+ available (e.g. when the ZEO server restarted).
+ </description>
+ </key>
+
</sectiontype>
</component>
Modified: ZODB/trunk/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/trunk/src/ZEO/tests/testZEO.py 2008-09-10 22:13:32 UTC (rev 91046)
+++ ZODB/trunk/src/ZEO/tests/testZEO.py 2008-09-10 23:15:14 UTC (rev 91047)
@@ -60,6 +60,8 @@
class DummyDB:
def invalidate(self, *args):
pass
+ def invalidateCache(*unused):
+ pass
class OneTimeTests(unittest.TestCase):
@@ -145,6 +147,58 @@
self.assertNotEquals(ZODB.utils.z64, storage3.lastTransaction())
storage3.close()
+ def checkDropCacheRatherVerifyImplementation(self):
+ # As it is quite difficult to set things up such that the verification
+ # optimizations do not step in, we emulate both the cache
+ # as well as the server.
+ from ZODB.TimeStamp import TimeStamp
+ class CacheEmulator(object):
+ # the settings below would be inconsitent for a normal cache
+ # but they are sufficient for our test setup
+ def __len__(self): return 1 # claim not to be empty
+ def contents(self): return () # do not invalidate anything
+ def getLastTid(self): return
+ def close(self): pass
+ class ServerEmulator(object):
+ def verify(*unused): pass
+ def endZeoVerify(*unused): pass
+ def lastTransaction(*unused): pass
+ storage = self._storage
+ storage._cache = cache = CacheEmulator()
+ server = ServerEmulator()
+ # test the standard behaviour
+ self.assertEqual(storage.verify_cache(server), "full verification")
+ # test the "drop cache rather verify" behaviour
+ storage._drop_cache_rather_verify = True
+ self.assertEqual(storage.verify_cache(server), "cache dropped")
+ # verify that we got a new cache
+ self.assert_(cache != storage._cache)
+
+
+class ConfigurationTests(unittest.TestCase):
+ def checkDropCacheRatherVerifyConfiguration(self):
+ from ZODB.config import storageFromString
+ # the default is to do verification and not drop the cache
+ cs = storageFromString('''
+ <zeoclient>
+ server localhost:9090
+ wait false
+ </zeoclient>
+ ''')
+ self.assertEqual(cs._drop_cache_rather_verify, False)
+ cs.close()
+ # now for dropping
+ cs = storageFromString('''
+ <zeoclient>
+ server localhost:9090
+ wait false
+ drop-cache-rather-verify true
+ </zeoclient>
+ ''')
+ self.assertEqual(cs._drop_cache_rather_verify, True)
+ cs.close()
+
+
class GenericTests(
# Base class for all ZODB tests
StorageTestBase.StorageTestBase,
@@ -955,7 +1009,9 @@
test_classes = [FileStorageTests, FileStorageRecoveryTests,
MappingStorageTests, DemoStorageTests,
- BlobAdaptedFileStorageTests, BlobWritableCacheTests]
+ BlobAdaptedFileStorageTests, BlobWritableCacheTests,
+ ConfigurationTests,
+ ]
def test_suite():
suite = unittest.TestSuite()
Modified: ZODB/trunk/src/ZODB/component.xml
===================================================================
--- ZODB/trunk/src/ZODB/component.xml 2008-09-10 22:13:32 UTC (rev 91046)
+++ ZODB/trunk/src/ZODB/component.xml 2008-09-10 23:15:14 UTC (rev 91047)
@@ -163,6 +163,13 @@
that are accepted by this server.
</description>
</key>
+ <!-- DM 2008-05-15: added -->
+ <key name="drop-cache-rather-verify" datatype="boolean" default="off">
+ <description>
+ A flag indicating whether the client cache should be dropped
+ instead of an expensive verification.
+ </description>
+ </key>
</sectiontype>
<sectiontype name="demostorage" datatype=".DemoStorage"
Modified: ZODB/trunk/src/ZODB/config.py
===================================================================
--- ZODB/trunk/src/ZODB/config.py 2008-09-10 22:13:32 UTC (rev 91046)
+++ ZODB/trunk/src/ZODB/config.py 2008-09-10 23:15:14 UTC (rev 91047)
@@ -167,6 +167,7 @@
wait=self.config.wait,
read_only=self.config.read_only,
read_only_fallback=self.config.read_only_fallback,
+ drop_cache_rather_verify=self.config.drop_cache_rather_verify,
username=self.config.username,
password=self.config.password,
realm=self.config.realm)
More information about the Zodb-checkins
mailing list