[Zodb-checkins] SVN: ZODB/trunk/src/ZEO/ Added an optimization to make sure there's always a free block after
Jim Fulton
jim at zope.com
Thu Nov 13 16:20:25 EST 2008
Log message for revision 92916:
Added an optimization to make sure there's always a free block after
an allocated block so that the first free block is right after the
last allocated block so that, on restart, we always continue writing
after the last written block.
Changed:
U ZODB/trunk/src/ZEO/cache.py
U ZODB/trunk/src/ZEO/tests/test_cache.py
-=-
Modified: ZODB/trunk/src/ZEO/cache.py
===================================================================
--- ZODB/trunk/src/ZEO/cache.py 2008-11-13 19:37:46 UTC (rev 92915)
+++ ZODB/trunk/src/ZEO/cache.py 2008-11-13 21:20:25 UTC (rev 92916)
@@ -159,9 +159,9 @@
# a temp file will be created)
self.path = path
- # - `maxsize`: total size of the cache file, in bytes; this is
- # ignored path names an existing file; perhaps we should attempt
- # to change the cache size in that case
+ # - `maxsize`: total size of the cache file
+ # We set to the minimum size of less than the minimum.
+ size = max(size, ZEC_HEADER_SIZE)
self.maxsize = size
# The number of records in the cache.
@@ -249,9 +249,10 @@
self.current = ZODB.fsIndex.fsIndex()
self.noncurrent = BTrees.LOBTree.LOBTree()
l = 0
- ofs = ZEC_HEADER_SIZE
+ last = ofs = ZEC_HEADER_SIZE
first_free_offset = 0
current = self.current
+ status = ' '
while ofs < fsize:
seek(ofs)
status = read(1)
@@ -288,23 +289,24 @@
raise ValueError("unknown status byte value %s in client "
"cache file" % 0, hex(ord(status)))
- if ofs + size >= maxsize:
+ last = ofs
+ ofs += size
+
+ if ofs >= maxsize:
# Oops, the file was bigger before.
- if ofs+size > maxsize:
+ if ofs > maxsize:
# The last record is too big. Replace it with a smaller
# free record
- size = maxsize-ofs
- seek(ofs)
+ size = maxsize-last
+ seek(last)
if size > 4:
write('f'+pack(">I", size))
else:
write("012345"[size])
sync(f)
- ofs += size
+ ofs = maxsize
break
- ofs += size
-
if fsize < maxsize:
assert ofs==fsize
# Make sure the OS really saves enough bytes for the file.
@@ -319,7 +321,10 @@
write('f' + pack(">I", block_size))
seek(block_size-5, 1)
sync(self.f)
- first_free_offset = ofs
+
+ # There is always data to read and
+ assert last and status in ' f1234'
+ first_free_offset = last
else:
assert ofs==maxsize
if maxsize < fsize:
@@ -551,14 +556,19 @@
# 2nd-level ZEO cache got a much higher hit rate if "very large"
# objects simply weren't cached. For now, we ignore the request
# only if the entire cache file is too small to hold the object.
- if size > min(max_block_size, self.maxsize - ZEC_HEADER_SIZE):
+ if size >= min(max_block_size, self.maxsize - ZEC_HEADER_SIZE):
return
self._n_adds += 1
self._n_added_bytes += size
self._len += 1
- nfreebytes = self._makeroom(size)
+ # In the next line, we ask for an extra to make sure we always
+ # have a free block after the new alocated block. This free
+ # block acts as a ring pointer, so that on restart, we start
+ # where we left off.
+ nfreebytes = self._makeroom(size+1)
+
assert size <= nfreebytes, (size, nfreebytes)
excess = nfreebytes - size
# If there's any excess (which is likely), we need to record a
Modified: ZODB/trunk/src/ZEO/tests/test_cache.py
===================================================================
--- ZODB/trunk/src/ZEO/tests/test_cache.py 2008-11-13 19:37:46 UTC (rev 92915)
+++ ZODB/trunk/src/ZEO/tests/test_cache.py 2008-11-13 21:20:25 UTC (rev 92916)
@@ -229,7 +229,7 @@
data = 'x'
recsize = ZEO.cache.allocated_record_overhead+len(data)
- for extra in (0, 2, recsize-2):
+ for extra in (2, recsize-2):
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+100*recsize+extra)
@@ -251,8 +251,13 @@
set(range(small)))
for i in range(100, 110):
cache.store(p64(i), n1, None, data)
- self.assertEquals(len(cache), small)
- expected_oids = set(range(10, 50)+range(100, 110))
+
+ # We use small-1 below because an extra object gets
+ # evicted because of the optimization to assure that we
+ # always get a free block after a new allocated block.
+ expected_len = small - 1
+ self.assertEquals(len(cache), expected_len)
+ expected_oids = set(range(11, 50)+range(100, 110))
self.assertEquals(
set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
@@ -261,7 +266,7 @@
cache.close()
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+small*recsize+extra)
- self.assertEquals(len(cache), small)
+ self.assertEquals(len(cache), expected_len)
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
@@ -270,16 +275,20 @@
large = 150
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+large*recsize+extra)
- self.assertEquals(len(cache), small)
+ self.assertEquals(len(cache), expected_len)
self.assertEquals(os.path.getsize(
'cache'), ZEO.cache.ZEC_HEADER_SIZE+large*recsize+extra)
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
+
for i in range(200, 305):
cache.store(p64(i), n1, None, data)
- self.assertEquals(len(cache), large)
- expected_oids = set(range(10, 50)+range(105, 110)+range(200, 305))
+
+ # We use large-2 for the same reason we used small-1 above.
+ expected_len = large-2
+ self.assertEquals(len(cache), expected_len)
+ expected_oids = set(range(11, 50)+range(106, 110)+range(200, 305))
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
@@ -287,7 +296,7 @@
cache.close()
cache = ZEO.cache.ClientCache(
'cache', size=ZEO.cache.ZEC_HEADER_SIZE+large*recsize+extra)
- self.assertEquals(len(cache), large)
+ self.assertEquals(len(cache), expected_len)
self.assertEquals(set(u64(oid) for (oid, tid) in cache.contents()),
expected_oids)
More information about the Zodb-checkins
mailing list