[Zodb-checkins] CVS: Zope3/src/zodb/storage - file.py:1.23.2.2
Jeremy Hylton
jeremy@zope.com
Fri, 11 Apr 2003 13:35:15 -0400
Update of /cvs-repository/Zope3/src/zodb/storage
In directory cvs.zope.org:/tmp/cvs-serv32300
Modified Files:
Tag: jeremy-new-pack-branch
file.py
Log Message:
A minimally functional pack. PackAllRevisions test succeeds.
=== Zope3/src/zodb/storage/file.py 1.23.2.1 => 1.23.2.2 ===
--- Zope3/src/zodb/storage/file.py:1.23.2.1 Thu Apr 10 18:41:12 2003
+++ Zope3/src/zodb/storage/file.py Fri Apr 11 13:35:14 2003
@@ -1457,10 +1457,9 @@
self._commit_lock_acquire,
self._commit_lock_release)
try:
- t = p.pack()
- if t is None:
+ opos = p.pack()
+ if opos is None:
return
- opos, index, vindex, tindex, tvindex = t
oldpath = self._name + ".old"
self._file.close()
try:
@@ -1474,7 +1473,7 @@
# OK, we're beyond the point of no return
os.rename(self._name + '.pack', self._name)
self._file = open(self._name, 'r+b')
- self._initIndex(index, vindex, tindex, tvindex)
+ self._initIndex(p.index, p.vindex, p.tindex, p.tvindex)
self._pos = opos
self._save_index()
finally:
@@ -1572,7 +1571,7 @@
self._file.seek(pos - p + 8)
return self._file.read(1) != ' ' # XXX or == "p"?
- def _pack_index(self, index):
+ def createPackIndex(self):
"""Return packpos and an index of objects reachable at the pack time.
If the storage is empty or it has been packed to a later time,
@@ -1585,24 +1584,24 @@
packpos, maxoid, ltid = self._read_index(index, vindex, tindex,
self._stop, read_only=1)
if packpos == self._metadata_size:
- return None, None
+ return None
if self._redundant_pack(packpos):
- return None, None
+ return None
del vindex, tindex
# Now traverse all objects starting from the root and add
# them to pindex. Any object in index but not in pindex
# is unreachable and should not be copied.
rootl = [ZERO]
- pindex = fsIndex()
+ self.pindex = fsIndex()
while rootl:
oid = rootl.pop()
- if oid in pindex:
+ if oid in self.pindex:
continue
p, refs, v = self._loada(oid, index)
rootl.extend(refs)
- pindex[oid] = index[oid]
- return packpos, pindex
+ self.pindex[oid] = index[oid]
+ return packpos
def pack(self):
# Pack copies all data reachable at the pack time or later.
@@ -1618,7 +1617,11 @@
packing = True
file_end = os.stat(self._file.name)[stat.ST_SIZE]
- packpos, pindex = self._pack_index()
+ packpos = self.createPackIndex()
+ if packpos is None:
+ return None
+ # XXX must be sure that packpos is the end of the file
+ # or the position of the first record after the pack time
assert packpos <= file_end
# Setup the destination file and copy the metadata.
@@ -1626,17 +1629,15 @@
self._file.seek(0)
self.ofile.write(self._file.read(self._metadata_size))
- self.copyToPacktime(packpos)
- self.copyRest()
+ pos = self.copyToPacktime(packpos)
+ pos = self.copyRest(pos)
# OK, we've copied everything. Now we need to wrap things up.
self.ofile.flush()
self.ofile.close()
self._file.close()
- # XXX probably don't need to return indexes, since caller
- # has object
- return opos, self.index, self.vindex, self.tindex, self.tvindex
+ return pos
def isCurNonversion(self, h, nvpos, curpos):
"""Return True if h is current non-version data,
@@ -1666,57 +1667,86 @@
pos = self._metadata_size
new_pos = pos
- while pos <= packpos:
+ while pos < packpos:
th = self._read_txn_header(pos)
copy = False
tend = pos + th.tlen
pos += th.headerlen()
- while pos < tend:
- h = self._read_data_header(pos)
- cur_nonversion = False
-
- # If this data record isn't current, don't copy it.
- # If the current record is in a version, this may be
- # the current non-version data.
- curpos = pindex.get(h.oid, 0)
- if curpos != pos:
- if self.isCurNonversion(h, curpos):
- cur_nonversion = True
- else:
- pos += h.recordlen()
- continue
- # If we are going to copy any data, we need to copy
- # the transaction header. Note that we will need to
- # patch up the transaction length when we are done.
- if not copy:
- th.status = "p"
- s = th.asString()
- self.ofile.write(s)
- new_tpos = new_pos
- new_pos += len(s)
- copy = True
-
- if cur_nonversion:
- self.nvindex[h.oid] = new_pos
-
- if h.plen:
- refs = self._file.read(8 * h.nrefs)
- data = self._file.read(h.plen)
- else:
- # If a current record has a backpointer, fetch
- # refs and data from the backpointer. We need
- # to write the data in the new record.
- data, refs, serial, tid = self._loadBackTxn(h.oid, h.back)
+ new_tpos, pos = self.copyDataRecords(pos, tend, th)
- self.writeDataRecord(h, data, refs, new_pos)
- new_pos = self.ofile.tell()
+ if new_tpos:
+ new_pos = self.ofile.tell() + 8
+ tlen = new_pos - new_tpos - 8
+ # Update the transaction length
+ self.ofile.seek(new_tpos + 8)
+ self.ofile.write(p64(tlen))
+ self.ofile.seek(new_pos)
+ self.ofile.write(p64(tlen))
- if copy:
- # now fix up the transaction header
- pass
+ # skip the end-of-transaction redundant length
+ pos += 8
+
+ return new_pos
+
+ def copyDataRecords(self, pos, tend, th):
+ """Copy any current data records between pos and tend.
+
+ Returns position of txn header in output file and position
+ of next record in the input file.
+ If any data records are copied, also write txn header (th).
+ """
+ copy = False
+ new_tpos = 0
+ while pos < tend:
+ h = self._read_data_header(pos)
+ cur_nonversion = False
+
+ # If this data record isn't current, don't copy it.
+ # If the current record is in a version, this may be
+ # the current non-version data.
+ curpos = self.pindex.get(h.oid, 0)
+ if curpos != pos:
+ if self.isCurNonversion(h, curpos, pos):
+ cur_nonversion = True
+ else:
+ pos += h.recordlen()
+ continue
+ pos += h.recordlen()
+
+ # If we are going to copy any data, we need to copy
+ # the transaction header. Note that we will need to
+ # patch up the transaction length when we are done.
+ if not copy:
+ th.status = "p"
+ s = th.asString()
+ new_tpos = self.ofile.tell()
+ self.ofile.write(s)
+ new_pos = new_tpos + len(s)
+ copy = True
+ if cur_nonversion:
+ self.nvindex[h.oid] = new_pos
+
+ if h.plen:
+ refs = self._file.read(8 * h.nrefs)
+ data = self._file.read(h.plen)
+ else:
+ # If a current record has a backpointer, fetch
+ # refs and data from the backpointer. We need
+ # to write the data in the new record.
+ data, refs, serial, tid = self._loadBackTxn(h.oid, h.back)
+
+ self.writeDataRecord(h, data, refs, new_pos)
+ new_pos = self.ofile.tell()
+
+ return new_tpos, pos
+
+ def copyRest(self, pos):
+ # Do something
+ return pos
+
def writeDataRecord(self, h, data, refs, new_pos):
# Update the header to reflect current information, then write
# it to the output file.
@@ -2272,8 +2302,8 @@
fromString = classmethod(fromString)
def asString(self):
- s = struct.pack(DATA_HDR, self.oid, self.serial, self.tloc,
- self.vlen, self.plen, self.nrefs, self.back)
+ s = struct.pack(DATA_HDR, self.oid, self.serial, self.prev,
+ self.tloc, self.vlen, self.nrefs, self.plen)
if self.version:
v = struct.pack(">QQ", self.pnv, self.vprev)
return s + v + self.version