[Zodb-checkins] CVS: Zope/lib/python/ZODB - fsrecover.py:1.5.6.2
Jeremy Hylton
jeremy at zope.com
Fri May 30 17:00:39 EDT 2003
Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv31454
Modified Files:
Tag: Zope-2_6-branch
fsrecover.py
Log Message:
Cross-port fixes from the ZODB3-3_1-branch.
=== Zope/lib/python/ZODB/fsrecover.py 1.5.6.1 => 1.5.6.2 ===
--- Zope/lib/python/ZODB/fsrecover.py:1.5.6.1 Mon Mar 17 14:51:52 2003
+++ Zope/lib/python/ZODB/fsrecover.py Fri May 30 16:00:38 2003
@@ -11,8 +11,6 @@
# FOR A PARTICULAR PURPOSE
#
##############################################################################
-
-
"""Simple script for repairing damaged FileStorage files.
Usage: %s [-f] input output
@@ -84,220 +82,272 @@
import getopt, ZODB.FileStorage, struct, time
from struct import unpack
-from ZODB.utils import t32, p64, U64
+from ZODB.utils import t32, p64, u64
from ZODB.TimeStamp import TimeStamp
from cPickle import loads
from ZODB.FileStorage import RecordIterator
-class EOF(Exception): pass
class ErrorFound(Exception): pass
def error(mess, *args):
raise ErrorFound(mess % args)
-def read_transaction_header(file, pos, file_size):
+def read_txn_header(f, pos, file_size, outp, ltid):
# Read the transaction record
- seek=file.seek
- read=file.read
-
- seek(pos)
- h=read(23)
- if len(h) < 23: raise EOF
+ f.seek(pos)
+ h = f.read(23)
+ if len(h) < 23:
+ raise EOFError
tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
if el < 0: el=t32-el
- tl=U64(stl)
-
- if status=='c': raise EOF
+ tl = u64(stl)
- if pos+(tl+8) > file_size:
+ if pos + (tl + 8) > file_size:
error("bad transaction length at %s", pos)
- if status not in ' up':
- error('invalid status, %s, at %s', status, pos)
+ if tl < (23 + ul + dl + el):
+ error("invalid transaction length, %s, at %s", tl, pos)
- if tl < (23+ul+dl+el):
- error('invalid transaction length, %s, at %s', tl, pos)
+ if ltid and tid < ltid:
+ error("time-stamp reducation %s < %s, at %s", u64(tid), u64(ltid), pos)
- tpos=pos
- tend=tpos+tl
+ if status == "c":
+ truncate(f, pos, file_size, output)
+ raise EOFError
- if status=='u':
+ if status not in " up":
+ error("invalid status, %r, at %s", status, pos)
+
+ tpos = pos
+ tend = tpos + tl
+
+ if status == "u":
# Undone transaction, skip it
- seek(tend)
- h=read(8)
- if h != stl: error('inconsistent transaction length at %s', pos)
- pos=tend+8
- return pos, None
-
- pos=tpos+(23+ul+dl+el)
- user=read(ul)
- description=read(dl)
+ f.seek(tend)
+ h = f.read(8)
+ if h != stl:
+ error("inconsistent transaction length at %s", pos)
+ pos = tend + 8
+ return pos, None, tid
+
+ pos = tpos+(23+ul+dl+el)
+ user = f.read(ul)
+ description = f.read(dl)
if el:
- try: e=loads(read(el))
+ try: e=loads(f.read(el))
except: e={}
else: e={}
result = RecordIterator(tid, status, user, description, e, pos, tend,
- file, tpos)
- pos=tend
+ f, tpos)
+ pos = tend
# Read the (intentionally redundant) transaction length
- seek(pos)
- h=read(8)
+ f.seek(pos)
+ h = f.read(8)
if h != stl:
error("redundant transaction length check failed at %s", pos)
- pos=pos+8
+ pos += 8
- return pos, result
+ return pos, result, tid
-def scan(file, pos, file_size):
- seek=file.seek
- read=file.read
+def truncate(f, pos, file_size, outp):
+ """Copy data from pos to end of f to a .trNNN file."""
+
+ i = 0
while 1:
- seek(pos)
- data=read(8096)
- if not data: return 0
+ trname = outp + ".tr%d" % i
+ if os.path.exists(trname):
+ i += 1
+ tr = open(trname, "wb")
+ copy(f, tr, file_size - pos)
+ f.seek(pos)
+ tr.close()
+
+def copy(src, dst, n):
+ while n:
+ buf = src.read(8096)
+ if not buf:
+ break
+ if len(buf) > n:
+ buf = buf[:n]
+ dst.write(buf)
+ n -= len(buf)
+
+def scan(f, pos):
+ """Return a potential transaction location following pos in f.
+
+ This routine scans forward from pos looking for the last data
+ record in a transaction. A period '.' always occurs at the end of
+ a pickle, and an 8-byte transaction length follows the last
+ pickle. If a period is followed by a plausible 8-byte transaction
+ length, assume that we have found the end of a transaction.
+
+ The caller should try to verify that the returned location is
+ actually a transaction header.
+ """
+ while 1:
+ f.seek(pos)
+ data = f.read(8096)
+ if not data:
+ return 0
- s=0
+ s = 0
while 1:
- l=data.find('.', s)
+ l = data.find(".", s)
if l < 0:
- pos=pos+8096
+ pos += len(data)
break
- if l > 8080:
- pos = pos + l
+ # If we are less than 8 bytes from the end of the
+ # string, we need to read more data.
+ s = l + 1
+ if s > len(data) - 8:
+ pos += l
break
- s=l+1
- tl=U64(data[s:s+8])
+ tl = u64(data[s:s+8])
if tl < pos:
return pos + s + 8
def iprogress(i):
- if i%2: print '.',
- else: print (i/2)%10,
+ if i % 2:
+ print ".",
+ else:
+ print (i/2) % 10,
sys.stdout.flush()
def progress(p):
- for i in range(p): iprogress(i)
-
-def recover(argv=sys.argv):
+ for i in range(p):
+ iprogress(i)
+def main():
try:
- opts, (inp, outp) = getopt.getopt(argv[1:], 'fv:pP:')
- force = partial = verbose = 0
- pack = None
- for opt, v in opts:
- if opt == '-v': verbose = int(v)
- elif opt == '-p': partial=1
- elif opt == '-f': force=1
- elif opt == '-P': pack=time.time()-float(v)
-
-
- force = filter(lambda opt: opt[0]=='-f', opts)
- partial = filter(lambda opt: opt[0]=='-p', opts)
- verbose = filter(lambda opt: opt[0]=='-v', opts)
- verbose = verbose and int(verbose[0][1]) or 0
- print 'Recovering', inp, 'into', outp
- except:
+ opts, (inp, outp) = getopt.getopt(sys.argv[1:], "fv:pP:")
+ except getopt.error:
die()
print __doc__ % argv[0]
+
+ force = partial = verbose = 0
+ pack = None
+ for opt, v in opts:
+ if opt == "-v":
+ verbose = int(v)
+ elif opt == "-p":
+ partial = 1
+ elif opt == "-f":
+ force = 1
+ elif opt == "-P":
+ pack = time.time() - float(v)
+ recover(inp, outp, verbose, partial, force, pack)
+
+def recover(inp, outp, verbose=0, partial=0, force=0, pack=0):
+ print "Recovering", inp, "into", outp
if os.path.exists(outp) and not force:
die("%s exists" % outp)
- file=open(inp, "rb")
- seek=file.seek
- read=file.read
- if read(4) != ZODB.FileStorage.packed_version:
+ f = open(inp, "rb")
+ if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
- seek(0,2)
- file_size=file.tell()
+ f.seek(0,2)
+ file_size = f.tell()
- ofs=ZODB.FileStorage.FileStorage(outp, create=1)
- _ts=None
- ok=1
- prog1=0
- preindex={}; preget=preindex.get # waaaa
- undone=0
+ ofs = ZODB.FileStorage.FileStorage(outp, create=1)
+ _ts = None
+ ok = 1
+ prog1 = 0
+ undone = 0
- pos=4
+ pos = 4L
+ ltid = None
while pos:
-
try:
- npos, transaction = read_transaction_header(file, pos, file_size)
- except EOF:
+ npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
+ except EOFError:
break
- except:
- print "\n%s: %s\n" % sys.exc_info()[:2]
- if not verbose: progress(prog1)
- pos = scan(file, pos, file_size)
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except Exception, err:
+ print "error reading txn header:", err
+ if not verbose:
+ progress(prog1)
+ pos = scan(f, pos)
+ if verbose > 1:
+ print "looking for valid txn header at", pos
continue
+ ltid = tid
- if transaction is None:
+ if txn is None:
undone = undone + npos - pos
- pos=npos
+ pos = npos
continue
else:
- pos=npos
+ pos = npos
- tid=transaction.tid
+ tid = txn.tid
if _ts is None:
- _ts=TimeStamp(tid)
+ _ts = TimeStamp(tid)
else:
- t=TimeStamp(tid)
+ t = TimeStamp(tid)
if t <= _ts:
- if ok: print ('Time stamps out of order %s, %s' % (_ts, t))
- ok=0
- _ts=t.laterThan(_ts)
- tid=`_ts`
+ if ok:
+ print ("Time stamps out of order %s, %s" % (_ts, t))
+ ok = 0
+ _ts = t.laterThan(_ts)
+ tid = `_ts`
else:
_ts = t
if not ok:
- print ('Time stamps back in order %s' % (t))
- ok=1
-
- if verbose:
- print 'begin',
- if verbose > 1: print
- sys.stdout.flush()
+ print ("Time stamps back in order %s" % (t))
+ ok = 1
- ofs.tpc_begin(transaction, tid, transaction.status)
+ ofs.tpc_begin(txn, tid, txn.status)
if verbose:
- print 'begin', pos, _ts,
- if verbose > 1: print
+ print "begin", pos, _ts,
+ if verbose > 1:
+ print
sys.stdout.flush()
- nrec=0
+ nrec = 0
try:
- for r in transaction:
- oid=r.oid
- if verbose > 1: print U64(oid), r.version, len(r.data)
- pre=preget(oid, None)
- s=ofs.store(oid, pre, r.data, r.version, transaction)
- preindex[oid]=s
- nrec=nrec+1
- except:
+ for r in txn:
+ if verbose > 1:
+ if r.data is None:
+ l = "bp"
+ else:
+ l = len(r.data)
+
+ print "%7d %s %s" % (u64(r.oid), l, r.version)
+ s = ofs.restore(r.oid, r.serial, r.data, r.version,
+ r.data_txn, txn)
+ nrec += 1
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except Exception, err:
if partial and nrec:
- ofs._status='p'
- ofs.tpc_vote(transaction)
- ofs.tpc_finish(transaction)
- if verbose: print 'partial'
+ ofs._status = "p"
+ ofs.tpc_vote(txn)
+ ofs.tpc_finish(txn)
+ if verbose:
+ print "partial"
else:
- ofs.tpc_abort(transaction)
- print "\n%s: %s\n" % sys.exc_info()[:2]
- if not verbose: progress(prog1)
- pos = scan(file, pos, file_size)
+ ofs.tpc_abort(txn)
+ print "error copying transaction:", err
+ if not verbose:
+ progress(prog1)
+ pos = scan(f, pos)
+ if verbose > 1:
+ print "looking for valid txn header at", pos
else:
- ofs.tpc_vote(transaction)
- ofs.tpc_finish(transaction)
+ ofs.tpc_vote(txn)
+ ofs.tpc_finish(txn)
if verbose:
- print 'finish'
+ print "finish"
sys.stdout.flush()
if not verbose:
@@ -320,5 +370,6 @@
ofs.close()
+if __name__ == "__main__":
+ main()
-if __name__=='__main__': recover()
More information about the Zodb-checkins
mailing list