[Zodb-checkins] CVS: ZODB3/ZODB - FileStorage.py:1.108

Jeremy Hylton jeremy@zope.com
Fri, 18 Oct 2002 12:52:01 -0400


Update of /cvs-repository/ZODB3/ZODB
In directory cvs.zope.org:/tmp/cvs-serv22246

Modified Files:
	FileStorage.py 
Log Message:
Various small cleanups in FileStorage.

Share backpointer implementation between _loadBack() and
_loadBackPOS() to avoid duplicate code.

Simplify interface between FileIterator and RecordIterator.  Don't
pass unused arguments and don't pass nested tuples.

Use global variables to hold trans and data struct formats.

Reformat several docstrings.

Add whitespace to RecordIterator so that I can read it.


=== ZODB3/ZODB/FileStorage.py 1.107 => 1.108 ===
--- ZODB3/ZODB/FileStorage.py:1.107	Wed Oct 16 17:54:50 2002
+++ ZODB3/ZODB/FileStorage.py	Fri Oct 18 12:52:00 2002
@@ -148,11 +148,17 @@
 register_subsystem('ZODB FS')
 
 z64='\0'*8
+# the struct formats for the headers
+TRANS_HDR = ">8s8scHHH"
+DATA_HDR = ">8s8s8s8sH8s"
 # constants to support various header sizes
 TRANS_HDR_LEN = 23
 DATA_HDR_LEN = 42
 DATA_VERSION_HDR_LEN = 58
 
+assert struct.calcsize(TRANS_HDR) == TRANS_HDR_LEN
+assert struct.calcsize(DATA_HDR) == DATA_HDR_LEN
+
 def warn(message, *data):
     LOG('ZODB FS', WARNING, "%s  warn: %s\n" % (packed_version,
                                                 (message % data)))
@@ -175,21 +181,19 @@
 class FileStorageFormatError(FileStorageError):
     """Invalid file format
 
-    The format of the given file is not valid
+    The format of the given file is not valid.
     """
 
 class CorruptedFileStorageError(FileStorageError,
                                 POSException.StorageSystemError):
-    """Corrupted file storage
-    """
+    """Corrupted file storage."""
 
 class CorruptedTransactionError(CorruptedFileStorageError): pass
 class CorruptedDataError(CorruptedFileStorageError): pass
 
 class FileStorageQuotaError(FileStorageError,
                             POSException.StorageSystemError):
-    """File storage quota exceeded
-    """
+    """File storage quota exceeded."""
 
 packed_version='FS21'
 
@@ -365,7 +369,7 @@
             if pos < 4: return 0
             seek(pos)
             s = read(TRANS_HDR_LEN)
-            tid, stl, status, ul, dl, el = unpack(">8s8scHHH", s)
+            tid, stl, status, ul, dl, el = unpack(TRANS_HDR, s)
             if not ltid: ltid=tid
             if stl != rstl: return 0 # inconsistent lengths
             if status == 'u': continue # undone trans, search back
@@ -379,7 +383,7 @@
                 # Read the data records for this transaction
                 seek(opos)
                 h=read(DATA_HDR_LEN)
-                oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+                oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
                 tloc=U64(stloc)
                 plen=U64(splen)
 
@@ -549,7 +553,7 @@
         file.seek(pos)
         read=file.read
         h=read(DATA_HDR_LEN)
-        doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
+        doid,serial,prev,tloc,vlen,plen = unpack(DATA_HDR, h)
         if vlen:
             nv = read(8) != z64
             file.seek(8,1) # Skip previous version record pointer
@@ -594,7 +598,7 @@
         file.seek(pos)
         read = file.read
         h = read(DATA_HDR_LEN)
-        doid, serial, prev, tloc, vlen, plen = unpack(">8s8s8s8sH8s", h)
+        doid, serial, prev, tloc, vlen, plen = unpack(DATA_HDR, h)
         if doid != oid:
             raise CorruptedDataError, h
         if vlen:
@@ -633,7 +637,7 @@
             while 1:
                 seek(pos)
                 h=read(DATA_HDR_LEN)
-                doid,dserial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
+                doid,dserial,prev,tloc,vlen,plen = unpack(DATA_HDR, h)
                 if doid != oid: raise CorruptedDataError, h
                 if dserial == serial: break # Yeee ha!
                 # Keep looking for serial
@@ -662,13 +666,12 @@
             except KeyError:
                 raise POSKeyError(oid)
             file=self._file
-            seek=file.seek
-            seek(pos)
+            file.seek(pos)
             doid,serial,prev,tloc,vlen = unpack(">8s8s8s8sH", file.read(34))
             if doid != oid:
                 raise CorruptedDataError, pos
             if vlen:
-                seek(24,1) # skip plen, pnv, and pv
+                file.read(24) # skip plen, pnv, and pv
                 return file.read(vlen)
             return ''
         finally: self._lock_release()
@@ -686,7 +689,7 @@
             if old:
                 self._file.seek(old)
                 h=self._file.read(DATA_HDR_LEN)
-                doid,oserial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+                doid,oserial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
                 if doid != oid: raise CorruptedDataError, h
                 if vlen:
                     pnv=self._file.read(8) # non-version data pointer
@@ -710,7 +713,7 @@
             here=pos+(tfile.tell()+self._thl)
             self._tindex[oid]=here
             newserial=self._serial
-            write(pack(">8s8s8s8sH8s",
+            write(pack(DATA_HDR,
                        oid, newserial, p64(old), p64(pos),
                        len(version), p64(len(data))
                        )
@@ -784,7 +787,7 @@
                 if old:
                     self._file.seek(old)
                     h = self._file.read(42)
-                    doid, x, y, z, vlen, w = unpack(">8s8s8s8sH8s", h)
+                    doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
                     if doid != oid:
                         raise CorruptedDataError, h
                     # XXX assert versions match?
@@ -940,7 +943,7 @@
                 # Read the data records for this transaction
                 seek(pos)
                 h=read(DATA_HDR_LEN)
-                oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+                oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
                 plen=U64(splen)
                 prev=U64(sprev)
                 dlen=DATA_HDR_LEN+(plen or 8)
@@ -973,7 +976,7 @@
         read=file.read
         file.seek(pos)
         h=read(DATA_HDR_LEN)
-        roid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+        roid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
         if roid != oid: raise UndoError('Invalid undo transaction id')
         if vlen:
             read(16) # skip nv pointer and version previous pointer
@@ -996,7 +999,7 @@
         self._file.seek(pos)
         read=self._file.read
         h=read(DATA_HDR_LEN)
-        doid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+        doid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
         if vlen:
             h=read(16)
             return read(vlen), h[:8]
@@ -1171,7 +1174,7 @@
             self._file.seek(pos)
             h = self._file.read(DATA_HDR_LEN)
             oid, serial, sprev, stloc, vlen, splen = \
-                 struct.unpack(">8s8s8s8sH8s", h)
+                 struct.unpack(DATA_HDR, h)
             if failed(oid):
                 del failures[oid] # second chance!
             plen = U64(splen)
@@ -1192,7 +1195,7 @@
                 failures[oid] = v
             else:
                 plen = len(p)
-                self._tfile.write(pack(">8s8s8s8sH8s",
+                self._tfile.write(pack(DATA_HDR,
                                        oid, self._serial, p64(ipos),
                                        ostloc, len(v), p64(plen)))
                 if v:
@@ -1283,7 +1286,7 @@
                 if len(r) >= size: return r
                 seek(pos)
                 h=read(DATA_HDR_LEN)
-                doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
+                doid,serial,prev,tloc,vlen,plen = unpack(DATA_HDR, h)
                 prev=U64(prev)
 
                 if vlen:
@@ -1302,7 +1305,7 @@
 
                 seek(U64(tloc))
                 h=read(TRANS_HDR_LEN)
-                tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+                tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
                 user_name=read(ul)
                 description=read(dl)
                 if el: d=loads(read(el))
@@ -1467,7 +1470,7 @@
                 seek(pos)
                 h=read(TRANS_HDR_LEN)
                 if len(h) < TRANS_HDR_LEN: break
-                tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+                tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
                 if status=='c':
                     # Oops. we found a checkpoint flag.
                     break
@@ -1510,7 +1513,7 @@
                     seek(pos)
                     h=read(DATA_HDR_LEN)
                     oid,serial,sprev,stloc,vlen,splen = unpack(
-                        ">8s8s8s8sH8s", h)
+                        DATA_HDR, h)
                     plen=U64(splen)
                     dlen=DATA_HDR_LEN+(plen or 8)
 
@@ -1545,7 +1548,7 @@
                                 seek(ppos)
                                 ph=read(DATA_HDR_LEN)
                                 pdoid,ps,pp,pt,pvlen,pplen = unpack(
-                                    ">8s8s8s8sH8s", ph)
+                                    DATA_HDR, ph)
                                 if not pvlen:
                                     # The most current record is committed, so
                                     # we can toss this one
@@ -1614,7 +1617,7 @@
                             p=p64(p)
 
                     sprev=p64(index_get(oid, 0))
-                    write(pack(">8s8s8s8sH8s",
+                    write(pack(DATA_HDR,
                                oid,serial,sprev,p64(otpos),vlen,splen))
                     if vlen:
                         if not pnv:
@@ -1780,7 +1783,7 @@
         seek(pos)
         h=read(TRANS_HDR_LEN)
         if len(h) < TRANS_HDR_LEN: break
-        tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+        tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
         if status=='c': break # Oops. we found a checkpoint flag.
         tl=U64(stl)
         tpos=pos
@@ -1805,7 +1808,7 @@
             # Read the data records for this transaction
             seek(pos)
             h=read(DATA_HDR_LEN)
-            oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+            oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
             plen=U64(splen)
             dlen=DATA_HDR_LEN+(plen or 8)
 
@@ -1834,7 +1837,7 @@
             # WRITE
             seek(opos)
             sprev=p64(index_get(oid, 0))
-            write(pack(">8s8s8s8sH8s",
+            write(pack(DATA_HDR,
                        oid,serial,sprev,p64(otpos),vlen,splen))
             if vlen:
                 if not pnv: write(z64)
@@ -1960,7 +1963,7 @@
                 file.truncate()
             break
 
-        tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+        tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
         if el < 0: el=t32-el
 
         if tid <= ltid:
@@ -2027,7 +2030,7 @@
 
             seek(pos)
             h=read(DATA_HDR_LEN)
-            oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+            oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
             prev=U64(sprev)
             tloc=U64(stloc)
             plen=U64(splen)
@@ -2082,37 +2085,30 @@
     return pos, maxoid, ltid
 
 
-def _loadBack(file, oid, back):
+def _loadBack_impl(file, oid, back):
+    # shared implementation used by various _loadBack methods
     while 1:
         old = U64(back)
         if not old:
             raise POSKeyError(oid)
         file.seek(old)
-        h=file.read(DATA_HDR_LEN)
-        doid, serial, prev, tloc, vlen, plen = unpack(">8s8s8s8sH8s", h)
+        h = file.read(DATA_HDR_LEN)
+        doid, serial, prev, tloc, vlen, plen = unpack(DATA_HDR, h)
 
         if vlen:
             file.seek(vlen + 16, 1)
         if plen != z64:
-            return file.read(U64(plen)), serial
+            return file.read(U64(plen)), serial, old
         back = file.read(8) # We got a back pointer!
 
-def _loadBackPOS(file, oid, back):
-    """Return the position of the record containing the data used by
-    the record at the given position (back)."""
-    seek=file.seek
-    read=file.read
+def _loadBack(file, oid, back):
+    data, serial, old = _loadBack_impl(file, oid, back)
+    return data, serial
 
-    while 1:
-        old=U64(back)
-        if not old:
-            raise POSKeyError(oid)
-        seek(old)
-        h=read(DATA_HDR_LEN)
-        doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
-        if vlen: seek(vlen+16,1)
-        if plen != z64: return old
-        back=read(8) # We got a back pointer!
+def _loadBackPOS(file, oid, back):
+    """Return position of data record for backpointer."""
+    data, serial, old = _loadBack_impl(file, oid, back)
+    return old
 
 def _truncate(file, name, pos):
     seek=file.seek
@@ -2224,7 +2220,7 @@
             h=read(TRANS_HDR_LEN)
             if len(h) < TRANS_HDR_LEN: break
 
-            tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+            tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
             if el < 0: el=t32-el
 
             if tid <= self._ltid:
@@ -2292,23 +2288,18 @@
                 except: e={}
             else: e={}
 
-            result=RecordIterator(
-                tid, status, user, description, e,
-                pos, (tend, file, seek, read,
-                      tpos,
-                      )
-                )
-
-            pos=tend
+            result = RecordIterator(tid, status, user, description, e, pos,
+                                    tend, file, tpos)
+            pos = tend
 
             # Read the (intentionally redundant) transaction length
             seek(pos)
-            h=read(8)
+            h = read(8)
             if h != stl:
                 warn("%s redundant transaction length check failed at %s",
                      self._file.name, pos)
                 break
-            self._pos=pos+8
+            self._pos = pos + 8
 
             return result
 
@@ -2317,48 +2308,48 @@
 class RecordIterator(Iterator, BaseStorage.TransactionRecord):
     """Iterate over the transactions in a FileStorage file.
     """
-    def __init__(self, tid, status, user, desc, ext, pos, stuff):
-        self.tid=tid
-        self.status=status
-        self.user=user
-        self.description=desc
-        self._extension=ext
-        self._pos=pos
-        self._stuff = stuff
+    def __init__(self, tid, status, user, desc, ext, pos, tend, file, tpos):
+        self.tid = tid
+        self.status = status
+        self.user = user
+        self.description = desc
+        self._extension = ext
+        self._pos = pos
+        self._tend = tend
+        self._file = file
+        self._tpos = tpos
 
     def next(self, index=0):
-        name=''
         pos = self._pos
-        tend, file, seek, read, tpos = self._stuff
-        while pos < tend:
+        while pos < self._tend:
             # Read the data records for this transaction
-            seek(pos)
-            h=read(DATA_HDR_LEN)
-            oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
-            prev=U64(sprev)
-            tloc=U64(stloc)
-            plen=U64(splen)
+            self._file.seek(pos)
+            h = self._file.read(DATA_HDR_LEN)
+            oid, serial, sprev, stloc, vlen, splen = unpack(DATA_HDR, h)
+            prev = U64(sprev)
+            tloc = U64(stloc)
+            plen = U64(splen)
 
-            dlen=DATA_HDR_LEN+(plen or 8)
+            dlen = DATA_HDR_LEN + (plen or 8)
 
             if vlen:
-                dlen=dlen+(16+vlen)
-                seek(8,1)
-                pv=U64(read(8))
-                version=read(vlen)
+                dlen += (16 + vlen)
+                tmp = self._file.read(16)
+                pv = U64(tmp[8:16])
+                version = self._file.read(vlen)
             else:
-                version=''
+                version = ''
 
-            if pos+dlen > tend or tloc != tpos:
+            if pos + dlen > self._tend or tloc != self._tpos:
                 warn("%s data record exceeds transaction record at %s",
-                     name, pos)
+                     file.name, pos)
                 break
 
-            self._pos=pos+dlen
+            self._pos = pos + dlen
             if plen:
-                p = read(plen)
+                p = self._file.read(plen)
             else:
-                p = read(8)
+                p = self._file.read(8)
                 if p == z64:
                     # If the backpointer is 0 (encoded as z64), then
                     # this transaction undoes the object creation.  It
@@ -2368,7 +2359,7 @@
                     # this.
                     p = None
                 else:
-                    p = _loadBack(file, oid, p)[0]
+                    p = _loadBack(self._file, oid, p)[0]
 
             r = Record(oid, serial, version, p)
 
@@ -2377,8 +2368,7 @@
         raise IndexError, index
 
 class Record(BaseStorage.DataRecord):
-    """An abstract database record
-    """
+    """An abstract database record."""
     def __init__(self, *args):
         self.oid, self.serial, self.version, self.data = args
 
@@ -2414,7 +2404,7 @@
         self.pos -= U64(self.file.read(8)) + 8
         self.file.seek(self.pos)
         h = self.file.read(TRANS_HDR_LEN)
-        tid, tl, status, ul, dl, el = struct.unpack(">8s8scHHH", h)
+        tid, tl, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
         if tid < self.packt or status == 'p':
             self.stop = 1
             return None