[Zodb-checkins] SVN: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ feature implemented
Dieter Maurer
dieter at handshake.de
Thu Aug 28 11:10:04 EDT 2008
Log message for revision 90550:
feature implemented
Changed:
U ZODB/branches/dm-zeo_drop_cache_instead_verify/src/CHANGES.txt
U ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/ClientStorage.py
U ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/component.xml
U ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/tests/testZEO.py
U ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/component.xml
U ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/config.py
-=-
Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/CHANGES.txt
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/CHANGES.txt 2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/CHANGES.txt 2008-08-28 15:10:02 UTC (rev 90550)
@@ -8,6 +8,12 @@
New Features
------------
+- New `ClientStorage` configuration option `drop_cache_rather_verify`.
+ If this option is true then the ZEO client cache is dropped instead of
+ the long (unoptimized) verification. For large caches, setting this
+ option can avoid effective downtimes in the order of hours when
+ the connection to the ZEO server was interrupted for a longer time.
+
- Versions are no-longer supported.
- ZEO cache files can be larger than 4G. Note that older ZEO cache
Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/ClientStorage.py 2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/ClientStorage.py 2008-08-28 15:10:02 UTC (rev 90550)
@@ -117,6 +117,7 @@
wait_for_server_on_startup=None, # deprecated alias for wait
wait=None, wait_timeout=None,
read_only=0, read_only_fallback=0,
+ drop_cache_rather_verify=False,
username='', password='', realm=None,
blob_dir=None, shared_blob_dir=False):
"""ClientStorage constructor.
@@ -191,6 +192,9 @@
realm -- not documented.
+ drop_cache_rather_verify -- a flag indicating that the cache
+ should be dropped rather than expensively verified.
+
blob_dir -- directory path for blob data. 'blob data' is data that
is retrieved via the loadBlob API.
@@ -213,6 +217,14 @@
if debug:
log2("ClientStorage(): debug argument is no longer used")
+ # Remember some parameters for "_setupCache"
+ self._var_ = var
+ self._storage_ = storage
+ self._client_ = client
+ self._cache_size_ = cache_size
+
+ self._drop_cache_rather_verify = drop_cache_rather_verify
+
# wait defaults to True, but wait_for_server_on_startup overrides
# if not None
if wait_for_server_on_startup is not None:
@@ -336,15 +348,7 @@
else:
self.fshelper = None
- # Decide whether to use non-temporary files
- if client is not None:
- dir = var or os.getcwd()
- cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
- else:
- cache_path = None
- self._cache = self.ClientCacheClass(cache_path, size=cache_size)
- # TODO: maybe there's a better time to open the cache? Unclear.
- self._cache.open()
+ self._setupCache()
self._rpc_mgr = self.ConnectionManagerClass(addr, self,
tmin=min_disconnect_poll,
@@ -604,6 +608,28 @@
self._ready.set()
return "quick verification"
+ # This breaks some tests and probably is not worth special treatment
+## if not self._cache: # an empty cache
+## log2("No verification necessary (cache is empty)")
+## self._server = server
+## self._ready.set()
+## return "no verification"
+
+ # From this point on, we do not have complete information about
+ # the missed transactions.
+ # Therefore, we cannot reliably sanitize the "Connection" caches
+ # To be on the safe side, we flush these caches.
+ if self._db is not None:
+ self._db.invalidateCache()
+
+ if self._client_ is None and self._cache and self._drop_cache_rather_verify:
+ log2("dropping cache")
+ self._cache.close()
+ self._setupCache() # creates a new cache
+ self._server = server
+ self._ready.set()
+ return "cache dropped"
+
log2("Verifying cache")
# setup tempfile to hold zeoVerify results
self._verification_invalidations = []
@@ -1248,3 +1274,19 @@
invalidate = invalidateVerify
end = endVerify
Invalidate = invalidateTrans
+
+ def _setupCache(self):
+ '''create and open the cache.'''
+ # Decide whether to use non-temporary files
+ var = self._var_
+ storage = self._storage_
+ client = self._client_
+ cache_size = self._cache_size_
+ if client is not None:
+ dir = var or os.getcwd()
+ cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
+ else:
+ cache_path = None
+ self._cache = self.ClientCacheClass(cache_path, size=cache_size)
+ # TODO: maybe there's a better time to open the cache? Unclear.
+ self._cache.open()
Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/component.xml
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/component.xml 2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/component.xml 2008-08-28 15:10:02 UTC (rev 90550)
@@ -102,6 +102,16 @@
<metadefault>$INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid)</metadefault>
</key>
+ <!-- DM 2006-06-12: added option -->
+ <key name="drop-cache-rather-verify" datatype="boolean"
+ required="no" default="false">
+ <description>
+ indicates that the cache should be dropped rather than
+ verified when the verification optimization is not
+ available (e.g. when the ZEO server restarted).
+ </description>
+ </key>
+
</sectiontype>
</component>
Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/tests/testZEO.py 2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/tests/testZEO.py 2008-08-28 15:10:02 UTC (rev 90550)
@@ -61,6 +61,8 @@
class DummyDB:
def invalidate(self, *args):
pass
+ def invalidateCache(*unused):
+ pass
class OneTimeTests(unittest.TestCase):
@@ -147,6 +149,60 @@
storage3.close()
+ def checkDropCacheRatherVerifyImplementation(self):
+ # As it is quite difficult to set things up such that the verification
+ # optimizations do not step in, we emulate both the cache
+ # as well as the server.
+ from ZODB.TimeStamp import TimeStamp
+ class CacheEmulator(object):
+ # the settings below would be inconsitent for a normal cache
+ # but they are sufficient for our test setup
+ def __len__(self): return 1 # claim not to be empty
+ def contents(self): return () # do not invalidate anything
+ def getLastTid(self): return
+ def close(self): pass
+ class ServerEmulator(object):
+ def verify(*unused): pass
+ def endZeoVerify(*unused): pass
+ def lastTransaction(*unused): pass
+ storage = self._storage
+ storage._cache = cache = CacheEmulator()
+ server = ServerEmulator()
+ # test the standard behaviour
+ self.assertEqual(storage.verify_cache(server), "full verification")
+ # test the "drop cache rather verify" behaviour
+ storage._drop_cache_rather_verify = True
+ self.assertEqual(storage.verify_cache(server), "cache dropped")
+ # verify that we got a new cache
+ self.assert_(cache != storage._cache)
+
+
+class ConfigurationTests(unittest.TestCase):
+ def checkDropCacheRatherVerifyConfiguration(self):
+ from ZODB.config import storageFromString
+ # the default is to do verification and not drop the cache
+ cs = storageFromString('''
+ <zeoclient>
+ server localhost:9090
+ wait false
+ </zeoclient>
+ ''')
+ self.assertEqual(cs._drop_cache_rather_verify, False)
+ cs.close()
+ # now for dropping
+ cs = storageFromString('''
+ <zeoclient>
+ server localhost:9090
+ wait false
+ drop-cache-rather-verify true
+ </zeoclient>
+ ''')
+ self.assertEqual(cs._drop_cache_rather_verify, True)
+ cs.close()
+
+
+
+
def get_port():
"""Return a port that is not in use.
@@ -873,7 +929,9 @@
test_classes = [FileStorageTests, MappingStorageTests, DemoStorageTests,
- BlobAdaptedFileStorageTests, BlobWritableCacheTests]
+ BlobAdaptedFileStorageTests, BlobWritableCacheTests,
+ ConfigurationTests,
+ ]
def test_suite():
suite = unittest.TestSuite()
Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/component.xml
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/component.xml 2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/component.xml 2008-08-28 15:10:02 UTC (rev 90550)
@@ -163,6 +163,13 @@
that are accepted by this server.
</description>
</key>
+ <!-- DM 2008-05-15: added -->
+ <key name="drop-cache-rather-verify" datatype="boolean" default="off">
+ <description>
+ A flag indicating whether the client cache should be dropped
+ instead of an expensive verification.
+ </description>
+ </key>
</sectiontype>
<sectiontype name="demostorage" datatype=".DemoStorage"
Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/config.py
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/config.py 2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/config.py 2008-08-28 15:10:02 UTC (rev 90550)
@@ -163,6 +163,7 @@
wait=self.config.wait,
read_only=self.config.read_only,
read_only_fallback=self.config.read_only_fallback,
+ drop_cache_rather_verify=self.config.drop_cache_rather_verify,
username=self.config.username,
password=self.config.password,
realm=self.config.realm)
More information about the Zodb-checkins
mailing list