[Zodb-checkins] CVS: ZODB3/ZODB/tests - testFileStorage.py:1.19.8.2.2.3

[Jim Fulton]

Update of /cvs-repository/ZODB3/ZODB/tests
In directory cvs.zope.org:/tmp/cvs-serv2822/tests

Modified Files:
      Tag: ZODB3-fast-restart-branch
	testFileStorage.py 
Log Message:

Barry and Jim

Several startup performance improvements:

- When sanity checking saved indexes, only check a few objects
  in the last transaction, rather than chacking every object.
  Otherwise, really large transactions could cause the sanity check
  to take a long time (28 seconds for 320000 objects on my machine.)

- Changed to use fsIndex (BTree-based) indexes. This not only saves
  memory, but it also speeds index loading by a factor of 4.

  o Included code to automatically convert old dictionary indexes to
    use fsIndex.

- Save indexes on startup without an index.

- Periodically save indexes on commit when the number of records
  (including transaction records) written since the last save exceeds
  the number of objects in the database (as of the previous save).
  This is somewhat conservative, since it seems to take about 10 times
  longer to write and read an object in the index as it does to read a
  record. 



=== ZODB3/ZODB/tests/testFileStorage.py 1.19.8.2.2.2 => 1.19.8.2.2.3 ===
--- ZODB3/ZODB/tests/testFileStorage.py:1.19.8.2.2.2	Tue Dec 17 12:30:57 2002
+++ ZODB3/ZODB/tests/testFileStorage.py	Tue Dec 17 18:09:07 2002
@@ -67,6 +67,102 @@
         else:
             self.fail("expect long user field to raise error")
 
+    def check_use_fsIndex(self):
+        from ZODB.fsIndex import fsIndex
+
+        self.assertEqual(self._storage._index.__class__, fsIndex)
+
+    # XXX We could really use some tests for sanity checking
+
+    def check_conversion_to_fsIndex_not_if_readonly(self):
+
+        self.tearDown()
+
+        class OldFileStorage(ZODB.FileStorage.FileStorage):
+            def _newIndexes(self):
+                return {}, {}, {}, {}
+
+
+        from ZODB.fsIndex import fsIndex
+        
+        # Hack FileStorage to create dictionary indexes
+        self._storage = OldFileStorage('FileStorageTests.fs')
+
+        self.assertEqual(type(self._storage._index), type({}))
+        for i in range(10):
+            self._dostore()
+            
+        # Should save the index
+        self._storage.close()
+
+        self._storage = ZODB.FileStorage.FileStorage(
+            'FileStorageTests.fs', read_only=1)
+        self.assertEqual(type(self._storage._index), type({}))
+    
+    def check_conversion_to_fsIndex(self):
+
+        self.tearDown()
+
+        class OldFileStorage(ZODB.FileStorage.FileStorage):
+            def _newIndexes(self):
+                return {}, {}, {}, {}
+
+
+        from ZODB.fsIndex import fsIndex
+        
+        # Hack FileStorage to create dictionary indexes
+        self._storage = OldFileStorage('FileStorageTests.fs')
+
+        self.assertEqual(type(self._storage._index), type({}))
+        for i in range(10):
+            self._dostore()
+            
+        oldindex = self._storage._index.copy()
+            
+        # Should save the index
+        self._storage.close()
+
+        self._storage = ZODB.FileStorage.FileStorage('FileStorageTests.fs')
+        self.assertEqual(self._storage._index.__class__, fsIndex)
+        self.failUnless(self._storage._used_index)
+
+        index = {}
+        for k, v in self._storage._index.items():
+            index[k] = v
+
+        self.assertEqual(index, oldindex)
+
+
+    def check_save_after_load_with_no_index(self):
+        for i in range(10):
+            self._dostore()
+        self._storage.close()
+        os.remove('FileStorageTests.fs.index')
+        self.open()
+        self.assertEqual(self._storage._saved, 1)
+
+
+    # This would make the unit tests too slow
+    # check_save_after_load_that_worked_hard(self)        
+
+    def check_periodic_save_index(self):
+
+        # Check the basic algorithm
+        oldsaved = self._storage._saved
+        self._storage._records_before_save = 10
+        for i in range(4):
+            self._dostore()
+        self.assertEqual(self._storage._saved, oldsaved)
+        self._dostore()
+        self.assertEqual(self._storage._saved, oldsaved+1)
+
+        # Now make sure the parameter changes as we get bigger
+        for i in range(20):
+            self._dostore()
+
+        self.failUnless(self._storage._records_before_save > 20)
+        
+
 class FileStorageRecoveryTest(
     StorageTestBase.StorageTestBase,
     IteratorStorage.IteratorDeepCompare,