[Zodb-checkins] CVS: ZODB3/ZEO - ClientCache.py:1.38.2.2 README.txt:1.3.10.1
Guido van Rossum
guido@python.org
Thu, 21 Nov 2002 13:58:51 -0500
Update of /cvs-repository/ZODB3/ZEO
In directory cvs.zope.org:/tmp/cvs-serv31932/ZEO
Modified Files:
Tag: ZODB3-3_1-branch
ClientCache.py README.txt
Log Message:
Merge ClientCache changes from trunk (make things methods, log storage name, use storage name in trace filename).
=== ZODB3/ZEO/ClientCache.py 1.38.2.1 => 1.38.2.2 ===
--- ZODB3/ZEO/ClientCache.py:1.38.2.1 Wed Oct 16 17:44:05 2002
+++ ZODB3/ZEO/ClientCache.py Thu Nov 21 13:58:51 2002
@@ -111,9 +111,6 @@
import zLOG
from ZEO.ICache import ICache
-def log(msg, level=zLOG.INFO):
- zLOG.LOG("ZEC", level, msg)
-
magic='ZEC0'
class ClientCache:
@@ -122,11 +119,14 @@
def __init__(self, storage='1', size=20000000, client=None, var=None):
# Arguments:
- # storage -- storage name (used in persistent cache file names only)
+ # storage -- storage name (used in filenames and log messages)
# size -- size limit in bytes of both files together
# client -- if not None, use a persistent cache file and use this name
# var -- directory where to create persistent cache files
+ self._storage = storage
+ self._limit = size / 2
+
# Allocate locks:
L = allocate_lock()
self._acquire = L.acquire
@@ -182,10 +182,9 @@
f[0].write(magic)
current = 0
- log("%s: storage=%r, size=%r; file[%r]=%r" %
- (self.__class__.__name__, storage, size, current, p[current]))
+ self.log("%s: storage=%r, size=%r; file[%r]=%r" %
+ (self.__class__.__name__, storage, size, current, p[current]))
- self._limit = size / 2
self._current = current
self._setup_trace()
@@ -203,8 +202,8 @@
f = self._f
current = self._current
if f[not current] is not None:
- read_index(index, serial, f[not current], not current)
- self._pos = read_index(index, serial, f[current], current)
+ self.read_index(serial, not current)
+ self._pos = self.read_index(serial, current)
return serial.items()
finally:
@@ -240,15 +239,15 @@
f.seek(ap)
h = f.read(27)
if len(h) != 27:
- log("invalidate: short record for oid %16x "
- "at position %d in cache file %d"
- % (U64(oid), ap, p < 0))
+ self.log("invalidate: short record for oid %16x "
+ "at position %d in cache file %d"
+ % (U64(oid), ap, p < 0))
del self._index[oid]
return None
if h[:8] != oid:
- log("invalidate: oid mismatch: expected %16x read %16x "
- "at position %d in cache file %d"
- % (U64(oid), U64(h[:8]), ap, p < 0))
+ self.log("invalidate: oid mismatch: expected %16x read %16x "
+ "at position %d in cache file %d"
+ % (U64(oid), U64(h[:8]), ap, p < 0))
del self._index[oid]
return None
f.seek(ap+8) # Switch from reading to writing
@@ -281,9 +280,9 @@
else:
tlen = -1
if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen:
- log("load: bad record for oid %16x "
- "at position %d in cache file %d"
- % (U64(oid), ap, p < 0))
+ self.log("load: bad record for oid %16x "
+ "at position %d in cache file %d"
+ % (U64(oid), ap, p < 0))
del self._index[oid]
return None
@@ -452,9 +451,9 @@
else:
tlen = -1
if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen:
- log("modifiedInVersion: bad record for oid %16x "
- "at position %d in cache file %d"
- % (U64(oid), ap, p < 0))
+ self.log("modifiedInVersion: bad record for oid %16x "
+ "at position %d in cache file %d"
+ % (U64(oid), ap, p < 0))
del self._index[oid]
return None
@@ -481,7 +480,7 @@
current = not self._current
self._current = current
self._trace(0x70)
- log("flipping cache files. new current = %d" % current)
+ self.log("flipping cache files. new current = %d" % current)
# Delete the half of the index that's no longer valid
index = self._index
for oid in index.keys():
@@ -551,19 +550,21 @@
def _setup_trace(self):
# See if cache tracing is requested through $ZEO_CACHE_TRACE.
+ # A dash and the storage name are appended to get the filename.
# If not, or if we can't write to the trace file,
# disable tracing by setting self._trace to a dummy function.
self._tracefile = None
tfn = os.environ.get("ZEO_CACHE_TRACE")
if tfn:
+ tfn = tfn + "-" + self._storage
try:
self._tracefile = open(tfn, "ab")
self._trace(0x00)
except IOError, msg:
self._tracefile = None
- log("cannot write tracefile %s (%s)" % (tfn, msg))
+ self.log("cannot write tracefile %s (%s)" % (tfn, msg))
else:
- log("opened tracefile %s" % tfn)
+ self.log("opened tracefile %s" % tfn)
if self._tracefile is None:
def notrace(*args):
pass
@@ -587,85 +588,90 @@
oid,
serial))
-def read_index(index, serial, f, fileindex):
- seek = f.seek
- read = f.read
- pos = 4
- count = 0
+ def read_index(self, serial, fileindex):
+ index = self._index
+ f = self._f[fileindex]
+ seek = f.seek
+ read = f.read
+ pos = 4
+ count = 0
- while 1:
- f.seek(pos)
- h = read(27)
- if len(h) != 27:
- # An empty read is expected, anything else is suspect
- if h:
- rilog("truncated header", pos, fileindex)
- break
-
- if h[8] in 'vni':
- tlen, vlen, dlen = unpack(">iHi", h[9:19])
- else:
- tlen = -1
- if tlen <= 0 or vlen < 0 or dlen < 0 or vlen + dlen > tlen:
- rilog("invalid header data", pos, fileindex)
- break
-
- oid = h[:8]
-
- if h[8] == 'v' and vlen:
- seek(dlen+vlen, 1)
- vdlen = read(4)
- if len(vdlen) != 4:
- rilog("truncated record", pos, fileindex)
- break
- vdlen = unpack(">i", vdlen)[0]
- if vlen+dlen+43+vdlen != tlen:
- rilog("inconsistent lengths", pos, fileindex)
+ while 1:
+ f.seek(pos)
+ h = read(27)
+ if len(h) != 27:
+ # An empty read is expected, anything else is suspect
+ if h:
+ self.rilog("truncated header", pos, fileindex)
break
- seek(vdlen, 1)
- vs = read(8)
- if read(4) != h[9:13]:
- rilog("inconsistent tlen", pos, fileindex)
+
+ if h[8] in 'vni':
+ tlen, vlen, dlen = unpack(">iHi", h[9:19])
+ else:
+ tlen = -1
+ if tlen <= 0 or vlen < 0 or dlen < 0 or vlen + dlen > tlen:
+ self.rilog("invalid header data", pos, fileindex)
break
- else:
- if h[8] in 'vn' and vlen == 0:
- if dlen+31 != tlen:
- rilog("inconsistent nv lengths", pos, fileindex)
- seek(dlen, 1)
+
+ oid = h[:8]
+
+ if h[8] == 'v' and vlen:
+ seek(dlen+vlen, 1)
+ vdlen = read(4)
+ if len(vdlen) != 4:
+ self.rilog("truncated record", pos, fileindex)
+ break
+ vdlen = unpack(">i", vdlen)[0]
+ if vlen+dlen+43+vdlen != tlen:
+ self.rilog("inconsistent lengths", pos, fileindex)
+ break
+ seek(vdlen, 1)
+ vs = read(8)
if read(4) != h[9:13]:
- rilog("inconsistent nv tlen", pos, fileindex)
+ self.rilog("inconsistent tlen", pos, fileindex)
break
- vs = None
-
- if h[8] in 'vn':
- if fileindex:
- index[oid] = -pos
else:
- index[oid] = pos
- serial[oid] = h[-8:], vs
- else:
- if serial.has_key(oid):
- # We have a record for this oid, but it was invalidated!
- del serial[oid]
- del index[oid]
-
-
- pos = pos + tlen
- count += 1
-
- f.seek(pos)
- try:
- f.truncate()
- except:
- pass
-
- if count:
- log("read_index: cache file %d has %d records and %d bytes"
- % (fileindex, count, pos))
-
- return pos
-
-def rilog(msg, pos, fileindex):
- # Helper to log messages from read_index
- log("read_index: %s at position %d in cache file %d"
- % (msg, pos, fileindex))
+ if h[8] in 'vn' and vlen == 0:
+ if dlen+31 != tlen:
+ self.rilog("inconsistent nv lengths", pos, fileindex)
+ seek(dlen, 1)
+ if read(4) != h[9:13]:
+ self.rilog("inconsistent nv tlen", pos, fileindex)
+ break
+ vs = None
+
+ if h[8] in 'vn':
+ if fileindex:
+ index[oid] = -pos
+ else:
+ index[oid] = pos
+ serial[oid] = h[-8:], vs
+ else:
+ if serial.has_key(oid):
+ # We have a record for this oid, but it was invalidated!
+ del serial[oid]
+ del index[oid]
+
+
+ pos = pos + tlen
+ count += 1
+
+ f.seek(pos)
+ try:
+ f.truncate()
+ except:
+ pass
+
+ if count:
+ self.log("read_index: cache file %d has %d records and %d bytes"
+ % (fileindex, count, pos))
+
+ return pos
+
+ def rilog(self, msg, pos, fileindex):
+ # Helper to log messages from read_index
+ self.log("read_index: %s at position %d in cache file %d"
+ % (msg, pos, fileindex))
+
+ def log(self, msg, level=zLOG.INFO):
+ zLOG.LOG("ZEC:%s" % self._storage, level, msg)
=== ZODB3/ZEO/README.txt 1.3 => 1.3.10.1 ===
--- ZODB3/ZEO/README.txt:1.3 Wed Sep 11 17:25:39 2002
+++ ZODB3/ZEO/README.txt Thu Nov 21 13:58:51 2002
@@ -59,11 +59,13 @@
----------------------
To enable cache tracing, set the environment variable ZEO_CACHE_TRACE
-to the name of a file to which the ZEO client process can write. If
-the file doesn't exist, the ZEO will try to create it. If there are
-problems with the file, a log message is written to the standard Zope
-log file. To start or stop tracing, the ZEO client process (typically
-a Zope application server) must be restarted.
+to the name of a file to which the ZEO client process can write. ZEO
+will append a hyphen and the storage name to the filename, to
+distinguish different storages. If the file doesn't exist, the ZEO
+will try to create it. If there are problems with the file, a log
+message is written to the standard Zope log file. To start or stop
+tracing, the ZEO client process (typically a Zope application server)
+must be restarted.
The trace file can grow pretty quickly; on a moderately loaded server,
we observed it growing by 5 MB per hour. The file consists of binary