[Zodb-checkins] CVS: Zope/lib/python/ZODB - fsIndex.py:1.1.2.1 FileStorage.py:1.60.4.2.8.1

[Jim Fulton]

Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv16184

Modified Files:
      Tag: BTreeFSIndex-branch
	FileStorage.py 
Added Files:
      Tag: BTreeFSIndex-branch
	fsIndex.py 
Log Message:
Experimental fsIndex that uses a specialized BTree to reduce memory usage by an order of magnitude.

=== Added File Zope/lib/python/ZODB/fsIndex.py ===
######################################################################
# Mapping from 8-character string to 8-character string
# where the first two bytes of every string must be two nulls.
######################################################################

try:
    from BTrees._fsBTree import fsBucket as _fsBTree
    #from BTrees.OOBTree import OOBTree as _fsBTree
except ImportError:
    def fsIndex(): return {}
else:

    from utils import U64, p64

    class fsIndex:

        def __init__(self):
            self._data={}

        def __getitem__(self, key):
            return U64('\0\0'+self._data[key[:-2]][key[-2:]])

        def get(self, key, default=None):
            v=self._data.get(key[:-2], self)
            if v is self: return default
            v=v.get(key[-2:], self)
            if v is self: return default
            return U64('\0\0'+v)

        def __setitem__(self, key, value):
            value=p64(value)
            if value[:2] != '\0\0':
                raise ValueError('Values too large')
            tree=self._data.get(key[:-2], self)
            if tree is self:
                tree = _fsBTree()
                self._data[key[:-2]]=tree
            tree[key[-2:]]=value[2:]

        def __len__(self):
            r=0
            for tree in self._data.values(): r+=len(tree)
            return r

        def update(self, mapping):
            for k, v in mapping.items():
                self[k]=v

        def has_key(self, key):
            v=self.get(key, self)
            return v is not self

        def clear(self): self._data.clear()

        def keys(self):
            r=[]
            for prefix, tree in self._data.items():
                for suffix, v in tree.items():
                    r.append(prefix+suffix)
            return r

        def items(self):
            r=[]
            for prefix, tree in self._data.items():
                for suffix, v in tree.items():
                    r.append(((prefix+suffix), U64('\0\0'+v)))
            return r

        def values(self):
            r=[]
            for prefix, tree in self._data.items():
                for suffix, v in tree.items():
                    r.append(U64('\0\0',v))
            return r


=== Zope/lib/python/ZODB/FileStorage.py 1.60.4.2 => 1.60.4.2.8.1 ===
 import ConflictResolution
 
+from fsIndex import fsIndex
+
 try: from posix import fsync
 except: fsync=None
 
@@ -333,7 +335,8 @@
 
     def __len__(self): return len(self._index)
 
-    def _newIndexes(self): return {}, {}, {}, {}
+    def _newIndexes(self):
+        return fsIndex(), {}, {}, {}
         
     def abortVersion(self, src, transaction):
         return self.commitVersion(src, '', transaction, abort=1)