[Zope3-checkins] CVS: Zope3/src/zodb/storage - file.py:1.23.2.3

[Jeremy Hylton]

Update of /cvs-repository/Zope3/src/zodb/storage
In directory cvs.zope.org:/tmp/cvs-serv1009/storage

Modified Files:
      Tag: jeremy-new-pack-branch
	file.py 
Log Message:
Completed rewrite of pack.  

The code is still a mess, but it passes the tests.  Need to sit on it
for a while, then do a code review.

Add testPackVersionsInPast() which failed with the old pack.  Still
need to get Dieter's failing test included.


=== Zope3/src/zodb/storage/file.py 1.23.2.2 => 1.23.2.3 === (885/985 lines abridged)
--- Zope3/src/zodb/storage/file.py:1.23.2.2	Fri Apr 11 13:35:14 2003
+++ Zope3/src/zodb/storage/file.py	Mon Apr 14 18:53:38 2003
@@ -444,11 +444,6 @@
             h.back = u64(self._file.read(8))
         return h
 
-    def _write_data_header(self, file, oid, serial, prev, tloc, vlen, nrefs,
-                           plen):
-        s = struct.pack(DATA_HDR, oid, serial, prev, tloc, vlen, nrefs, plen)
-        file.write(s)
-
     def _write_version_header(self, file, pnv, vprev, version):
         s = struct.pack(">QQ", pnv, vprev)
         file.write(s + version)
@@ -509,7 +504,156 @@
     def __init__(self, afile):
         self._file = afile
 
-class FileStorage(BaseStorage, FileStorageFormatter):
+class DataCopier(FileStorageFormatter):
+    """Mixin class for copying transactions into a storage.
+
+    The restore() and pack() methods share a need to copy data records
+    and update pointers to data in earlier transaction records.  This
+    class provides the shared logic.
+
+    The mixin extends the FileStorageFormatter with a copy() method.
+    It also requires that the concrete class provides the following
+    attributes:
+
+    _file -- file with earlier destination data
+    _tfile -- destination file for copied data
+    _packt -- p64() representation of latest pack time
+    _pos -- file pos of destination transaction
+    _tindex -- maps oid to data record file pos
+    _tvindex -- maps version name to data record file pos
+
+    _tindex and _tvindex are updated by copy().
+
+    The copy() method does not do any locking.
+    """
+
+    def _txn_find(self, tid, stop_at_pack):
+        # _pos always points just past the last transaction
+        pos = self._pos
+        while pos > 1024:
+            self._file.seek(pos - 8)
+            pos = pos - u64(self._file.read(8)) - 8
+            self._file.seek(pos)
+            h = self._file.read(TRANS_HDR_LEN)

[-=- -=- -=- 885 lines omitted -=- -=- -=-]

+        self._file.seek(self.pos - 8)
+        self.pos -= u64(self._file.read(8)) + 8
+        if self.pos < 1024:
             return None
-        if status != ' ':
+        h = self._read_txn_header(self.pos)
+        if h.tid < self.packt or h.status == 'p':
+            self.stop = 1
             return None
-        d = u = ''
-        # user and description are utf-8 encoded strings
-        if ul:
-            u = self.file.read(ul).decode('utf-8')
-        if dl:
-            d = self.file.read(dl).decode('utf-8')
-        e = {}
-        if el:
-            try:
-                e = loads(self.file.read(el))
-            except:
-                pass
-        d = {'id': base64.encodestring(tid).rstrip(),
-             'time': TimeStamp(tid).timeTime(),
-             'user_name': u,
-             'description': d}
-        d.update(e)
+        assert h.status == " "
+        d = {'id': base64.encodestring(h.tid).rstrip(),
+             'time': TimeStamp(h.tid).timeTime(),
+             'user_name': h.user,
+             'description': h.descr}
+        if h.ext:
+            ext = loads(h.ext)
+            d.update(ext)
         return d
 
 class DataHeader:
@@ -2309,6 +2213,12 @@
             return s + v + self.version
         else:
             return s
+
+    def setVersion(self, version, pnv, vprev):
+        self.version = version
+        self.vlen = len(version)
+        self.pnv = pnv
+        self.vprev = vprev
 
     def parseVersion(self, buf):
         self.pnv, self.vprev = struct.unpack(">QQ", buf[:16])