[Zodb-checkins] CVS: Zope/lib/python/ZODB - coptimizations.c:1.17.60.5 FileStorage.py:1.98.2.4 BaseStorage.py:1.21.2.3
Jeremy Hylton
jeremy@zope.com
Wed, 18 Dec 2002 16:59:20 -0500
Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv13244/lib/python/ZODB
Modified Files:
Tag: Zope-2_6-branch
coptimizations.c FileStorage.py BaseStorage.py
Log Message:
Merge latest round of changes from ZODB3-3_1-branch
=== Zope/lib/python/ZODB/coptimizations.c 1.17.60.4 => 1.17.60.5 ===
--- Zope/lib/python/ZODB/coptimizations.c:1.17.60.4 Fri Dec 13 16:40:37 2002
+++ Zope/lib/python/ZODB/coptimizations.c Wed Dec 18 16:59:19 2002
@@ -156,6 +156,22 @@
return NULL;
}
+/* persistent_id_call()
+
+ Returns a reference to a persistent object, appending it to the the
+ persistent_id's list of objects. If a non-persistent object is
+ found, return None.
+
+ The returned reference can be either class info, oid pair or a
+ plain old oid. If it is a pair, the class info is the module and
+ the name of the class. The class info can be used to create a
+ ghost without loading the class.
+
+ For unusual objects, e.g. ZClasses, return just the oid. An object
+ is unusual if it isn't an ExtensionClass, because that means it
+ doesn't inherit from Persistence, or if it has __getinitargs__().
+*/
+
static PyObject *
persistent_id_call(persistent_id *self, PyObject *args, PyObject *kwargs)
{
=== Zope/lib/python/ZODB/FileStorage.py 1.98.2.3 => 1.98.2.4 ===
--- Zope/lib/python/ZODB/FileStorage.py:1.98.2.3 Wed Dec 11 11:30:45 2002
+++ Zope/lib/python/ZODB/FileStorage.py Wed Dec 18 16:59:19 2002
@@ -202,6 +202,8 @@
# default pack time is 0
_packt = z64
+ _records_before_save = 10000
+
def __init__(self, file_name, create=0, read_only=0, stop=None,
quota=None):
@@ -269,7 +271,10 @@
r = self._restore_index()
if r is not None:
+ self._used_index = 1 # Marker for testing
+
index, vindex, start, maxoid, ltid = r
+
self._initIndex(index, vindex, tindex, tvindex)
self._pos, self._oid, tid = read_index(
self._file, file_name, index, vindex, tindex, stop,
@@ -277,10 +282,16 @@
read_only=read_only,
)
else:
+ self._used_index = 0 # Marker for testing
self._pos, self._oid, tid = read_index(
self._file, file_name, index, vindex, tindex, stop,
read_only=read_only,
)
+ self._save_index()
+
+ self._records_before_save = max(self._records_before_save,
+ len(self._index))
+
self._ltid = tid
self._ts = tid = TimeStamp(tid)
@@ -309,6 +320,7 @@
# hook to use something other than builtin dict
return {}, {}, {}, {}
+ _saved = 0
def _save_index(self):
"""Write the database index to a file to support quick startup
"""
@@ -325,6 +337,7 @@
p.dump(info)
f.flush()
f.close()
+
try:
try:
os.remove(index_name)
@@ -333,6 +346,8 @@
os.rename(tmp_name, index_name)
except: pass
+ self._saved += 1
+
def _clear_index(self):
index_name=self.__name__+'.index'
if os.path.exists(index_name):
@@ -350,22 +365,29 @@
object positions cause zero to be returned.
"""
- if pos < 100: return 0
- file=self._file
- seek=file.seek
- read=file.read
+ if pos < 100:
+ return 0 # insane
+ file = self._file
+ seek = file.seek
+ read = file.read
seek(0,2)
- if file.tell() < pos: return 0
- ltid=None
+ if file.tell() < pos:
+ return 0 # insane
+ ltid = None
+
+ max_checked = 5
+ checked = 0
- while 1:
+ while checked < max_checked:
seek(pos-8)
- rstl=read(8)
- tl=U64(rstl)
- pos=pos-tl-8
- if pos < 4: return 0
+ rstl = read(8)
+ tl = U64(rstl)
+ pos = pos-tl-8
+ if pos < 4:
+ return 0 # insane
seek(pos)
s = read(TRANS_HDR_LEN)
+<<<<<<< FileStorage.py
tid, stl, status, ul, dl, el = unpack(TRANS_HDR, s)
if not ltid: ltid=tid
if stl != rstl: return 0 # inconsistent lengths
@@ -375,23 +397,52 @@
tend=pos+tl
opos=pos+(TRANS_HDR_LEN + ul + dl + el)
if opos==tend: continue # empty trans
+=======
+ tid, stl, status, ul, dl, el = unpack(TRANS_HDR, s)
+ if not ltid:
+ ltid = tid
+ if stl != rstl:
+ return 0 # inconsistent lengths
+ if status == 'u':
+ continue # undone trans, search back
+ if status not in ' p':
+ return 0 # insane
+ if tl < (TRANS_HDR_LEN + ul + dl + el):
+ return 0 # insane
+ tend = pos+tl
+ opos = pos+(TRANS_HDR_LEN + ul + dl + el)
+ if opos == tend:
+ continue # empty trans
+>>>>>>> 1.105.2.11
- while opos < tend:
+ while opos < tend and checked < max_checked:
# Read the data records for this transaction
seek(opos)
+<<<<<<< FileStorage.py
h=read(DATA_HDR_LEN)
oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
tloc=U64(stloc)
plen=U64(splen)
+=======
+ h = read(DATA_HDR_LEN)
+ oid, serial, sprev, stloc, vlen, splen = unpack(DATA_HDR, h)
+ tloc = U64(stloc)
+ plen = U64(splen)
+
+ dlen = DATA_HDR_LEN+(plen or 8)
+ if vlen:
+ dlen = dlen+(16+vlen)
+>>>>>>> 1.105.2.11
- dlen=DATA_HDR_LEN+(plen or 8)
- if vlen: dlen=dlen+(16+vlen)
+ if opos+dlen > tend or tloc != pos:
+ return 0 # insane
- if opos+dlen > tend or tloc != pos: return 0
+ if index.get(oid, 0) != opos:
+ return 0 # insane
- if index.get(oid, 0) != opos: return 0
+ checked += 1
- opos=opos+dlen
+ opos = opos+dlen
return ltid
@@ -420,7 +471,6 @@
if index is None or pos is None or oid is None or vindex is None:
return None
pos = long(pos)
-
tid=self._sane(index, pos)
if not tid: return None
@@ -946,6 +996,9 @@
finally:
self._lock_release()
+ # Keep track of the number of records that we've written
+ _records_written = 0
+
def _finish(self, tid, u, d, e):
nextpos=self._nextpos
if nextpos:
@@ -962,6 +1015,16 @@
self._index.update(self._tindex)
self._vindex.update(self._tvindex)
+
+ # Update the number of records that we've written
+ # +1 for the transaction record
+ self._records_written += len(self._tindex) + 1
+ if self._records_written >= self._records_before_save:
+ self._save_index()
+ self._records_written = 0
+ self._records_before_save = max(self._records_before_save,
+ len(self._index))
+
self._ltid = tid
def _abort(self):
=== Zope/lib/python/ZODB/BaseStorage.py 1.21.2.2 => 1.21.2.3 ===
--- Zope/lib/python/ZODB/BaseStorage.py:1.21.2.2 Tue Nov 12 16:13:58 2002
+++ Zope/lib/python/ZODB/BaseStorage.py Wed Dec 18 16:59:19 2002
@@ -265,8 +265,8 @@
restoring = 1
else:
restoring = 0
- for transaction in other.iterator():
-
+ fiter = other.iterator()
+ for transaction in fiter:
tid=transaction.tid
if _ts is None:
_ts=TimeStamp(tid)
@@ -299,6 +299,8 @@
self.tpc_vote(transaction)
self.tpc_finish(transaction)
+
+ fiter.close()
class TransactionRecord:
"""Abstract base class for iterator protocol"""