[Zodb-checkins] SVN: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ feature implemented

Dieter Maurer dieter at handshake.de
Thu Aug 28 11:10:04 EDT 2008


Log message for revision 90550:
  feature implemented

Changed:
  U   ZODB/branches/dm-zeo_drop_cache_instead_verify/src/CHANGES.txt
  U   ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/ClientStorage.py
  U   ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/component.xml
  U   ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/tests/testZEO.py
  U   ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/component.xml
  U   ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/config.py

-=-
Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/CHANGES.txt
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/CHANGES.txt	2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/CHANGES.txt	2008-08-28 15:10:02 UTC (rev 90550)
@@ -8,6 +8,12 @@
 New Features
 ------------
 
+- New `ClientStorage` configuration option `drop_cache_rather_verify`.
+  If this option is true then the ZEO client cache is dropped instead of
+  the long (unoptimized) verification. For large caches, setting this
+  option can avoid effective downtimes in the order of hours when
+  the connection to the ZEO server was interrupted for a longer time.
+
 - Versions are no-longer supported.
 
 - ZEO cache files can be larger than 4G. Note that older ZEO cache

Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/ClientStorage.py	2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/ClientStorage.py	2008-08-28 15:10:02 UTC (rev 90550)
@@ -117,6 +117,7 @@
                  wait_for_server_on_startup=None, # deprecated alias for wait
                  wait=None, wait_timeout=None,
                  read_only=0, read_only_fallback=0,
+                 drop_cache_rather_verify=False,
                  username='', password='', realm=None,
                  blob_dir=None, shared_blob_dir=False):
         """ClientStorage constructor.
@@ -191,6 +192,9 @@
 
         realm -- not documented.
 
+        drop_cache_rather_verify -- a flag indicating that the cache
+            should be dropped rather than expensively verified.
+
         blob_dir -- directory path for blob data.  'blob data' is data that
             is retrieved via the loadBlob API.
 
@@ -213,6 +217,14 @@
         if debug:
             log2("ClientStorage(): debug argument is no longer used")
 
+        # Remember some parameters for "_setupCache"
+        self._var_ = var
+        self._storage_ = storage
+        self._client_ = client
+        self._cache_size_ = cache_size
+
+        self._drop_cache_rather_verify = drop_cache_rather_verify
+
         # wait defaults to True, but wait_for_server_on_startup overrides
         # if not None
         if wait_for_server_on_startup is not None:
@@ -336,15 +348,7 @@
         else:
             self.fshelper = None
 
-        # Decide whether to use non-temporary files
-        if client is not None:
-            dir = var or os.getcwd()
-            cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
-        else:
-            cache_path = None
-        self._cache = self.ClientCacheClass(cache_path, size=cache_size)
-        # TODO:  maybe there's a better time to open the cache?  Unclear.
-        self._cache.open()
+        self._setupCache()
 
         self._rpc_mgr = self.ConnectionManagerClass(addr, self,
                                                     tmin=min_disconnect_poll,
@@ -604,6 +608,28 @@
                 self._ready.set()
                 return "quick verification"
         
+        # This breaks some tests and probably is not worth special treatment
+##        if not self._cache: # an empty cache
+##            log2("No verification necessary (cache is empty)")
+##            self._server = server
+##            self._ready.set()
+##            return "no verification"
+
+        # From this point on, we do not have complete information about
+        # the missed transactions.
+        # Therefore, we cannot reliably sanitize the "Connection" caches
+        # To be on the safe side, we flush these caches.
+        if self._db is not None:
+            self._db.invalidateCache()
+
+        if self._client_ is None and self._cache and self._drop_cache_rather_verify:
+            log2("dropping cache")
+            self._cache.close()
+            self._setupCache() # creates a new cache
+            self._server = server
+            self._ready.set()
+            return "cache dropped"
+
         log2("Verifying cache")
         # setup tempfile to hold zeoVerify results
         self._verification_invalidations = []
@@ -1248,3 +1274,19 @@
     invalidate = invalidateVerify
     end = endVerify
     Invalidate = invalidateTrans
+
+    def _setupCache(self):
+        '''create and open the cache.'''
+        # Decide whether to use non-temporary files
+        var = self._var_
+        storage = self._storage_
+        client = self._client_
+        cache_size = self._cache_size_
+        if client is not None:
+            dir = var or os.getcwd()
+            cache_path = os.path.join(dir, "%s-%s.zec" % (client, storage))
+        else:
+            cache_path = None
+        self._cache = self.ClientCacheClass(cache_path, size=cache_size)
+        # TODO:  maybe there's a better time to open the cache?  Unclear.
+        self._cache.open()

Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/component.xml
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/component.xml	2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/component.xml	2008-08-28 15:10:02 UTC (rev 90550)
@@ -102,6 +102,16 @@
       <metadefault>$INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid)</metadefault>
     </key>
 
+    <!-- DM 2006-06-12: added option -->
+    <key name="drop-cache-rather-verify" datatype="boolean"
+         required="no" default="false">
+       <description>
+         indicates that the cache should be dropped rather than
+	 verified when the verification optimization is not
+	 available (e.g. when the ZEO server restarted).
+       </description>
+    </key>
+
   </sectiontype>
 
 </component>

Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/tests/testZEO.py	2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZEO/tests/testZEO.py	2008-08-28 15:10:02 UTC (rev 90550)
@@ -61,6 +61,8 @@
 class DummyDB:
     def invalidate(self, *args):
         pass
+    def invalidateCache(*unused):
+        pass
 
 
 class OneTimeTests(unittest.TestCase):
@@ -147,6 +149,60 @@
         storage3.close()
 
 
+    def checkDropCacheRatherVerifyImplementation(self):
+        # As it is quite difficult to set things up such that the verification
+        # optimizations do not step in, we emulate both the cache
+        # as well as the server.
+        from ZODB.TimeStamp import TimeStamp
+        class CacheEmulator(object):
+            # the settings below would be inconsitent for a normal cache
+            # but they are sufficient for our test setup
+            def __len__(self): return 1 # claim not to be empty
+            def contents(self): return () # do not invalidate anything
+            def getLastTid(self): return
+            def close(self): pass
+        class ServerEmulator(object):
+            def verify(*unused): pass
+            def endZeoVerify(*unused): pass
+            def lastTransaction(*unused): pass
+        storage = self._storage
+        storage._cache = cache = CacheEmulator()
+        server = ServerEmulator()
+        # test the standard behaviour
+        self.assertEqual(storage.verify_cache(server), "full verification")
+        # test the "drop cache rather verify" behaviour
+        storage._drop_cache_rather_verify = True
+        self.assertEqual(storage.verify_cache(server), "cache dropped")
+        # verify that we got a new cache
+        self.assert_(cache != storage._cache)
+
+
+class ConfigurationTests(unittest.TestCase):
+    def checkDropCacheRatherVerifyConfiguration(self):
+        from ZODB.config import storageFromString
+        # the default is to do verification and not drop the cache
+        cs = storageFromString('''
+        <zeoclient>
+          server localhost:9090
+          wait false
+        </zeoclient>
+        ''')
+        self.assertEqual(cs._drop_cache_rather_verify, False)
+        cs.close()
+        # now for dropping
+        cs = storageFromString('''
+        <zeoclient>
+          server localhost:9090
+          wait false
+          drop-cache-rather-verify true
+        </zeoclient>
+        ''')
+        self.assertEqual(cs._drop_cache_rather_verify, True)
+        cs.close()
+
+
+                
+
 def get_port():
     """Return a port that is not in use.
 
@@ -873,7 +929,9 @@
 
 
 test_classes = [FileStorageTests, MappingStorageTests, DemoStorageTests,
-                BlobAdaptedFileStorageTests, BlobWritableCacheTests]
+                BlobAdaptedFileStorageTests, BlobWritableCacheTests,
+                ConfigurationTests,
+                ]
 
 def test_suite():
     suite = unittest.TestSuite()

Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/component.xml
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/component.xml	2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/component.xml	2008-08-28 15:10:02 UTC (rev 90550)
@@ -163,6 +163,13 @@
         that are accepted by this server.
       </description>
     </key>
+    <!-- DM 2008-05-15: added -->
+    <key name="drop-cache-rather-verify" datatype="boolean" default="off">
+      <description>
+        A flag indicating whether the client cache should be dropped
+        instead of an expensive verification.
+      </description>
+    </key>
   </sectiontype>
 
   <sectiontype name="demostorage" datatype=".DemoStorage"

Modified: ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/config.py
===================================================================
--- ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/config.py	2008-08-28 14:17:52 UTC (rev 90549)
+++ ZODB/branches/dm-zeo_drop_cache_instead_verify/src/ZODB/config.py	2008-08-28 15:10:02 UTC (rev 90550)
@@ -163,6 +163,7 @@
             wait=self.config.wait,
             read_only=self.config.read_only,
             read_only_fallback=self.config.read_only_fallback,
+            drop_cache_rather_verify=self.config.drop_cache_rather_verify,
             username=self.config.username,
             password=self.config.password,
             realm=self.config.realm)



More information about the Zodb-checkins mailing list