[Zope-Checkins] CVS: Zope/lib/python/ZODB - fsrecover.py:1.5.6.2

Jeremy Hylton jeremy@zope.com
Fri, 30 May 2003 16:00:39 -0400


Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv31454

Modified Files:
      Tag: Zope-2_6-branch
	fsrecover.py 
Log Message:
Cross-port fixes from the ZODB3-3_1-branch.


=== Zope/lib/python/ZODB/fsrecover.py 1.5.6.1 => 1.5.6.2 ===
--- Zope/lib/python/ZODB/fsrecover.py:1.5.6.1	Mon Mar 17 14:51:52 2003
+++ Zope/lib/python/ZODB/fsrecover.py	Fri May 30 16:00:38 2003
@@ -11,8 +11,6 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
-
-
 """Simple script for repairing damaged FileStorage files.
 
 Usage: %s [-f] input output
@@ -84,220 +82,272 @@
 
 import getopt, ZODB.FileStorage, struct, time
 from struct import unpack
-from ZODB.utils import t32, p64, U64
+from ZODB.utils import t32, p64, u64
 from ZODB.TimeStamp import TimeStamp
 from cPickle import loads
 from ZODB.FileStorage import RecordIterator
 
-class EOF(Exception): pass
 class ErrorFound(Exception): pass
 
 def error(mess, *args):
     raise ErrorFound(mess % args)
 
-def read_transaction_header(file, pos, file_size):
+def read_txn_header(f, pos, file_size, outp, ltid):
     # Read the transaction record
-    seek=file.seek
-    read=file.read
-
-    seek(pos)
-    h=read(23)
-    if len(h) < 23: raise EOF
+    f.seek(pos)
+    h = f.read(23)
+    if len(h) < 23:
+        raise EOFError
 
     tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
     if el < 0: el=t32-el
 
-    tl=U64(stl)
-
-    if status=='c': raise EOF
+    tl = u64(stl)
 
-    if pos+(tl+8) > file_size:
+    if pos + (tl + 8) > file_size:
         error("bad transaction length at %s", pos)
 
-    if status not in ' up':
-        error('invalid status, %s, at %s', status, pos)
+    if tl < (23 + ul + dl + el):
+        error("invalid transaction length, %s, at %s", tl, pos)
 
-    if tl < (23+ul+dl+el):
-        error('invalid transaction length, %s, at %s', tl, pos)
+    if ltid and tid < ltid:
+        error("time-stamp reducation %s < %s, at %s", u64(tid), u64(ltid), pos)
 
-    tpos=pos
-    tend=tpos+tl
+    if status == "c":
+        truncate(f, pos, file_size, output)
+        raise EOFError
 
-    if status=='u':
+    if status not in " up":
+        error("invalid status, %r, at %s", status, pos)
+
+    tpos = pos
+    tend = tpos + tl
+
+    if status == "u":
         # Undone transaction, skip it
-        seek(tend)
-        h=read(8)
-        if h != stl: error('inconsistent transaction length at %s', pos)
-        pos=tend+8
-        return pos, None
-
-    pos=tpos+(23+ul+dl+el)
-    user=read(ul)
-    description=read(dl)
+        f.seek(tend)
+        h = f.read(8)
+        if h != stl:
+            error("inconsistent transaction length at %s", pos)
+        pos = tend + 8
+        return pos, None, tid
+
+    pos = tpos+(23+ul+dl+el)
+    user = f.read(ul)
+    description = f.read(dl)
     if el:
-        try: e=loads(read(el))
+        try: e=loads(f.read(el))
         except: e={}
     else: e={}
 
     result = RecordIterator(tid, status, user, description, e, pos, tend,
-                            file, tpos)
-    pos=tend
+                            f, tpos)
+    pos = tend
 
     # Read the (intentionally redundant) transaction length
-    seek(pos)
-    h=read(8)
+    f.seek(pos)
+    h = f.read(8)
     if h != stl:
         error("redundant transaction length check failed at %s", pos)
-    pos=pos+8
+    pos += 8
 
-    return pos, result
+    return pos, result, tid
 
-def scan(file, pos, file_size):
-    seek=file.seek
-    read=file.read
+def truncate(f, pos, file_size, outp):
+    """Copy data from pos to end of f to a .trNNN file."""
+
+    i = 0
     while 1:
-        seek(pos)
-        data=read(8096)
-        if not data: return 0
+        trname = outp + ".tr%d" % i
+        if os.path.exists(trname):
+            i += 1
+    tr = open(trname, "wb")
+    copy(f, tr, file_size - pos)
+    f.seek(pos)
+    tr.close()
+
+def copy(src, dst, n):
+    while n:
+        buf = src.read(8096)
+        if not buf:
+            break
+        if len(buf) > n:
+            buf = buf[:n]
+        dst.write(buf)
+        n -= len(buf)
+
+def scan(f, pos):
+    """Return a potential transaction location following pos in f.
+
+    This routine scans forward from pos looking for the last data
+    record in a transaction.  A period '.' always occurs at the end of
+    a pickle, and an 8-byte transaction length follows the last
+    pickle.  If a period is followed by a plausible 8-byte transaction
+    length, assume that we have found the end of a transaction.
+
+    The caller should try to verify that the returned location is
+    actually a transaction header.
+    """
+    while 1:
+        f.seek(pos)
+        data = f.read(8096)
+        if not data:
+            return 0
 
-        s=0
+        s = 0
         while 1:
-            l=data.find('.', s)
+            l = data.find(".", s)
             if l < 0:
-                pos=pos+8096
+                pos += len(data)
                 break
-            if l > 8080:
-                pos = pos + l
+            # If we are less than 8 bytes from the end of the
+            # string, we need to read more data.
+            s = l + 1
+            if s > len(data) - 8:
+                pos += l
                 break
-            s=l+1
-            tl=U64(data[s:s+8])
+            tl = u64(data[s:s+8])
             if tl < pos:
                 return pos + s + 8
 
 def iprogress(i):
-    if i%2: print '.',
-    else: print (i/2)%10,
+    if i % 2:
+        print ".",
+    else:
+        print (i/2) % 10,
     sys.stdout.flush()
 
 def progress(p):
-    for i in range(p): iprogress(i)
-
-def recover(argv=sys.argv):
+    for i in range(p):
+        iprogress(i)
 
+def main():
     try:
-        opts, (inp, outp) = getopt.getopt(argv[1:], 'fv:pP:')
-        force = partial = verbose = 0
-        pack = None
-        for opt, v in opts:
-            if opt == '-v': verbose = int(v)
-            elif opt == '-p': partial=1
-            elif opt == '-f': force=1
-            elif opt == '-P': pack=time.time()-float(v)
-
-
-        force = filter(lambda opt: opt[0]=='-f', opts)
-        partial = filter(lambda opt: opt[0]=='-p', opts)
-        verbose = filter(lambda opt: opt[0]=='-v', opts)
-        verbose = verbose and int(verbose[0][1]) or 0
-        print 'Recovering', inp, 'into', outp
-    except:
+        opts, (inp, outp) = getopt.getopt(sys.argv[1:], "fv:pP:")
+    except getopt.error:
         die()
         print __doc__ % argv[0]
+        
+    force = partial = verbose = 0
+    pack = None
+    for opt, v in opts:
+        if opt == "-v":
+            verbose = int(v)
+        elif opt == "-p":
+            partial = 1
+        elif opt == "-f":
+            force = 1
+        elif opt == "-P":
+            pack = time.time() - float(v)
 
+    recover(inp, outp, verbose, partial, force, pack)
+
+def recover(inp, outp, verbose=0, partial=0, force=0, pack=0):
+    print "Recovering", inp, "into", outp
 
     if os.path.exists(outp) and not force:
         die("%s exists" % outp)
 
-    file=open(inp, "rb")
-    seek=file.seek
-    read=file.read
-    if read(4) != ZODB.FileStorage.packed_version:
+    f = open(inp, "rb")
+    if f.read(4) != ZODB.FileStorage.packed_version:
         die("input is not a file storage")
 
-    seek(0,2)
-    file_size=file.tell()
+    f.seek(0,2)
+    file_size = f.tell()
 
-    ofs=ZODB.FileStorage.FileStorage(outp, create=1)
-    _ts=None
-    ok=1
-    prog1=0
-    preindex={}; preget=preindex.get   # waaaa
-    undone=0
+    ofs = ZODB.FileStorage.FileStorage(outp, create=1)
+    _ts = None
+    ok = 1
+    prog1 = 0
+    undone = 0
 
-    pos=4
+    pos = 4L
+    ltid = None
     while pos:
-
         try:
-            npos, transaction = read_transaction_header(file, pos, file_size)
-        except EOF:
+            npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
+        except EOFError:
             break
-        except:
-            print "\n%s: %s\n" % sys.exc_info()[:2]
-            if not verbose: progress(prog1)
-            pos = scan(file, pos, file_size)
+        except (KeyboardInterrupt, SystemExit):
+            raise
+        except Exception, err:
+            print "error reading txn header:", err
+            if not verbose:
+                progress(prog1)
+            pos = scan(f, pos)
+            if verbose > 1:
+                print "looking for valid txn header at", pos
             continue
+        ltid = tid
 
-        if transaction is None:
+        if txn is None:
             undone = undone + npos - pos
-            pos=npos
+            pos = npos
             continue
         else:
-            pos=npos
+            pos = npos
 
-        tid=transaction.tid
+        tid = txn.tid
 
         if _ts is None:
-            _ts=TimeStamp(tid)
+            _ts = TimeStamp(tid)
         else:
-            t=TimeStamp(tid)
+            t = TimeStamp(tid)
             if t <= _ts:
-                if ok: print ('Time stamps out of order %s, %s' % (_ts, t))
-                ok=0
-                _ts=t.laterThan(_ts)
-                tid=`_ts`
+                if ok:
+                    print ("Time stamps out of order %s, %s" % (_ts, t))
+                ok = 0
+                _ts = t.laterThan(_ts)
+                tid = `_ts`
             else:
                 _ts = t
                 if not ok:
-                    print ('Time stamps back in order %s' % (t))
-                    ok=1
-
-        if verbose:
-            print 'begin',
-            if verbose > 1: print
-            sys.stdout.flush()
+                    print ("Time stamps back in order %s" % (t))
+                    ok = 1
 
-        ofs.tpc_begin(transaction, tid, transaction.status)
+        ofs.tpc_begin(txn, tid, txn.status)
 
         if verbose:
-            print 'begin', pos, _ts,
-            if verbose > 1: print
+            print "begin", pos, _ts,
+            if verbose > 1:
+                print
             sys.stdout.flush()
 
-        nrec=0
+        nrec = 0
         try:
-            for r in transaction:
-                oid=r.oid
-                if verbose > 1: print U64(oid), r.version, len(r.data)
-                pre=preget(oid, None)
-                s=ofs.store(oid, pre, r.data, r.version, transaction)
-                preindex[oid]=s
-                nrec=nrec+1
-        except:
+            for r in txn:
+                if verbose > 1:
+                    if r.data is None:
+                        l = "bp"
+                    else:
+                        l = len(r.data)
+                        
+                    print "%7d %s %s" % (u64(r.oid), l, r.version)
+                s = ofs.restore(r.oid, r.serial, r.data, r.version,
+                                r.data_txn, txn)
+                nrec += 1
+        except (KeyboardInterrupt, SystemExit):
+            raise
+        except Exception, err:
             if partial and nrec:
-                ofs._status='p'
-                ofs.tpc_vote(transaction)
-                ofs.tpc_finish(transaction)
-                if verbose: print 'partial'
+                ofs._status = "p"
+                ofs.tpc_vote(txn)
+                ofs.tpc_finish(txn)
+                if verbose:
+                    print "partial"
             else:
-                ofs.tpc_abort(transaction)
-            print "\n%s: %s\n" % sys.exc_info()[:2]
-            if not verbose: progress(prog1)
-            pos = scan(file, pos, file_size)
+                ofs.tpc_abort(txn)
+            print "error copying transaction:", err
+            if not verbose:
+                progress(prog1)
+            pos = scan(f, pos)
+            if verbose > 1:
+                print "looking for valid txn header at", pos
         else:
-            ofs.tpc_vote(transaction)
-            ofs.tpc_finish(transaction)
+            ofs.tpc_vote(txn)
+            ofs.tpc_finish(txn)
             if verbose:
-                print 'finish'
+                print "finish"
                 sys.stdout.flush()
 
         if not verbose:
@@ -320,5 +370,6 @@
 
     ofs.close()
 
+if __name__ == "__main__":
+    main()
 
-if __name__=='__main__': recover()