[Zodb-checkins] SVN: ZODB/trunk/src/ Bug Fixed:

[Jim Fulton]

Log message for revision 100808:
  Bug Fixed:
    Saving indexes for large file storages failed (with the error:
    RuntimeError: maximum recursion depth exceeded).  This can cause a
    FileStorage to fail to start because it gets an error trying to save
    its index.
  

Changed:
  U   ZODB/trunk/src/CHANGES.txt
  U   ZODB/trunk/src/ZODB/FileStorage/FileStorage.py
  U   ZODB/trunk/src/ZODB/FileStorage/tests.py

-=-
Modified: ZODB/trunk/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt	2009-06-10 18:14:53 UTC (rev 100807)
+++ ZODB/trunk/src/CHANGES.txt	2009-06-10 18:42:24 UTC (rev 100808)
@@ -8,6 +8,11 @@
 Bugs Fixed
 ----------
 
+- Saving indexes for large file storages failed (with the error:
+  RuntimeError: maximum recursion depth exceeded).  This can cause a
+  FileStorage to fail to start because it gets an error trying to save
+  its index.
+
 - Sizes of new objects weren't added to the object cache size
   estimation, causing the object-cache size limiting feature to let
   the cache grow too large when many objects were added.

Modified: ZODB/trunk/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/FileStorage/FileStorage.py	2009-06-10 18:14:53 UTC (rev 100807)
+++ ZODB/trunk/src/ZODB/FileStorage/FileStorage.py	2009-06-10 18:42:24 UTC (rev 100808)
@@ -33,6 +33,7 @@
 from ZODB.utils import p64, u64, z64
 
 import base64
+import BTrees.OOBTree
 import errno
 import logging
 import os
@@ -248,8 +249,17 @@
         f=open(tmp_name,'wb')
         p=Pickler(f,1)
 
-        info={'index': self._index, 'pos': self._pos}
+        # Pickle the index buckets first to avoid deep recursion:
+        buckets = []
+        bucket = self._index._data._firstbucket
+        while bucket is not None:
+            buckets.append(bucket)
+            bucket = bucket._next
+        buckets.reverse()
 
+        info=BTrees.OOBTree.Bucket(dict(
+            _buckets=buckets, index=self._index, pos=self._pos))
+
         p.dump(info)
         f.flush()
         f.close()

Modified: ZODB/trunk/src/ZODB/FileStorage/tests.py
===================================================================
--- ZODB/trunk/src/ZODB/FileStorage/tests.py	2009-06-10 18:14:53 UTC (rev 100807)
+++ ZODB/trunk/src/ZODB/FileStorage/tests.py	2009-06-10 18:42:24 UTC (rev 100808)
@@ -125,6 +125,49 @@
     >>> db.close()
     """
 
+def _save_index():
+    """
+
+_save_index can fail for large indexes.
+
+    >>> import ZODB.utils
+    >>> fs = ZODB.FileStorage.FileStorage('data.fs')
+
+    >>> t = transaction.begin()
+    >>> fs.tpc_begin(t)
+    >>> oid = 0
+    >>> for i in range(5000):
+    ...     oid += (1<<16)
+    ...     _ = fs.store(ZODB.utils.p64(oid), ZODB.utils.z64, 'x', '', t)
+    >>> fs.tpc_vote(t)
+    >>> fs.tpc_finish(t)
+
+    >>> import sys
+    >>> old_limit = sys.getrecursionlimit()
+    >>> sys.setrecursionlimit(50)
+    >>> fs._save_index()
+
+Make sure we can restore:
+
+    >>> import logging
+    >>> handler = logging.StreamHandler(sys.stdout)
+    >>> logger = logging.getLogger('ZODB.FileStorage')
+    >>> logger.setLevel(logging.DEBUG)
+    >>> logger.addHandler(handler)
+    >>> index, pos, tid = fs._restore_index()
+    >>> index.items() == fs._index.items()
+    True
+    >>> pos, tid = fs._pos, fs._tid
+
+cleanup
+
+    >>> logger.setLevel(logging.NOTSET)
+    >>> logger.removeHandler(handler)
+    >>> sys.setrecursionlimit(old_limit)
+
+    """
+
+
 def test_suite():
     return unittest.TestSuite((
         doctest.DocFileSuite(