[Zodb-checkins] SVN: ZODB/branches/jim-dev/src/Z checkpoint
Jim Fulton
jim at zope.com
Thu Dec 4 14:46:22 EST 2008
Log message for revision 93630:
checkpoint
Changed:
U ZODB/branches/jim-dev/src/ZEO/ClientStorage.py
U ZODB/branches/jim-dev/src/ZEO/tests/testZEO.py
U ZODB/branches/jim-dev/src/ZODB/component.xml
U ZODB/branches/jim-dev/src/ZODB/config.py
-=-
Modified: ZODB/branches/jim-dev/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/jim-dev/src/ZEO/ClientStorage.py 2008-12-04 18:51:29 UTC (rev 93629)
+++ ZODB/branches/jim-dev/src/ZEO/ClientStorage.py 2008-12-04 19:46:22 UTC (rev 93630)
@@ -33,6 +33,7 @@
import cPickle
import logging
import os
+import re
import socket
import stat
import sys
@@ -121,7 +122,7 @@
drop_cache_rather_verify=False,
username='', password='', realm=None,
blob_dir=None, shared_blob_dir=False,
- blob_cache_size=1<<62, blob_cache_size_check=100,
+ blob_cache_size=None, blob_cache_size_check=100,
):
"""ClientStorage constructor.
@@ -221,13 +222,16 @@
zrpc.
blob_cache_size
- Maximum size of the ZEO cache, in bytes. Defaults to 1GB.
+ Maximum size of the ZEO blob cache, in bytes. If not set, then
+ the cache size isn't checked and the blob directory will
+ grow without bound.
+
This option is ignored if shared_blob_dir is true.
blob_cache_size_check
ZEO check size as percent of blob_cache_size. The ZEO
cache size will be checked when this many bytes have been
- loaded into the cache. Defaults to 50% of the blob cache
+ loaded into the cache. Defaults to 100% of the blob cache
size. This option is ignored if shared_blob_dir is true.
Note that the authentication protocol is defined by the server
@@ -390,10 +394,13 @@
self._cache = self.ClientCacheClass(cache_path, size=cache_size)
+
self._blob_cache_size = blob_cache_size
- self._blob_cache_size_check = (
- blob_cache_size * blob_cache_size_check / 100)
- self._check_blob_size()
+ self._blob_data_bytes_loaded = 0
+ if blob_cache_size is not None:
+ self._blob_cache_size_check = (
+ blob_cache_size * blob_cache_size_check / 100)
+ self._check_blob_size()
self._rpc_mgr = self.ConnectionManagerClass(addr, self,
tmin=min_disconnect_poll,
@@ -455,69 +462,24 @@
self._check_blob_size_thread.join()
_check_blob_size_thread = None
- def _check_blob_size(self):
- self._blob_data_bytes_loaded = 0
+ def _check_blob_size(self, bytes=None):
+ if self._blob_cache_size is None:
+ return
if self.shared_blob_dir or not self.blob_dir:
return
+ if (bytes is not None) and (bytes < self._blob_cache_size_check):
+ return
+
+ self._blob_data_bytes_loaded = 0
check_blob_size_thread = threading.Thread(
- target=self._check_blob_size_method)
+ target=_check_blob_cache_size,
+ args=(self.blob_dir, self._blob_cache_size),
+ )
check_blob_size_thread.setDaemon(True)
check_blob_size_thread.start()
self._check_blob_size_thread = check_blob_size_thread
- def _check_blob_size_method(self):
- try:
- check_lock = zc.lockfile.LockFile(
- os.path.join(self.blob_dir, 'cache.lock'))
- except zc.lockfile.LockError:
- # Someone is already cleaning up, so don't bother
- return
-
- try:
- target = self._blob_cache_size
- size = 0
- tmp = self.temporaryDirectory()
- blob_suffix = ZODB.blob.BLOB_SUFFIX
- files_by_atime = BTrees.IOBTree.BTree()
- for base, dirs, files in os.walk(self.blob_dir):
- if base == tmp:
- del dirs[:]
- continue
- for file_name in files:
- if not file_name.endswith(blob_suffix):
- continue
- file_name = os.path.join(base, file_name)
- stat = os.stat(file_name)
- size += stat.st_size
- t = stat.st_atime
- if t not in files_by_atime:
- files_by_atime[t] = []
- files_by_atime[t].append(file_name)
-
- while size > target and files_by_atime:
- for file_name in files_by_atime.pop(files_by_atime.minKey()):
- lockfilename = os.path.join(os.path.dirname(file_name),
- '.lock')
- try:
- lock = zc.lockfile.LockFile(lockfilename)
- except zc.lockfile.LockError:
- continue # In use, skip
-
- try:
- size = os.stat(file_name).st_size
- try:
- ZODB.blob.remove_committed(file_name)
- except OSError, v:
- pass # probably open on windows
- else:
- size -= size
- finally:
- lock.close()
- finally:
- check_lock.close()
-
-
def registerDB(self, db):
"""Storage API: register a database for invalidation messages.
@@ -997,8 +959,7 @@
f.write(chunk)
f.close()
self._blob_data_bytes_loaded += len(chunk)
- if self._blob_data_bytes_loaded > self._blob_cache_size_check:
- self._check_blob_size()
+ self._check_blob_size(self._blob_data_bytes_loaded)
def receiveBlobStop(self, oid, serial):
blob_filename = self.fshelper.getBlobFilename(oid, serial)
@@ -1022,7 +983,6 @@
# We're using a server shared cache. If the file isn't
# here, it's not anywhere.
raise POSException.POSKeyError("No blob file", oid, serial)
-
if os.path.exists(blob_filename):
return _accessed(blob_filename)
@@ -1253,8 +1213,7 @@
blobfilename,
self.fshelper.getBlobFilename(oid, tid),
)
- if self._blob_data_bytes_loaded > self._blob_cache_size_check:
- self._check_blob_size()
+ self._check_blob_size(self._blob_data_bytes_loaded)
self._tbuf.clear()
@@ -1647,3 +1606,70 @@
except OSError:
pass # We tried. :)
return filename
+
+cache_file_name = re.compile(r'\d+$').match
+def _check_blob_cache_size(blob_dir, target):
+
+ layout = open(os.path.join(blob_dir, ZODB.blob.LAYOUT_MARKER)
+ ).read().strip()
+ if not layout == 'zeocache':
+ raise ValueError("Invalid blob directory layout", layout)
+
+ try:
+ check_lock = zc.lockfile.LockFile(
+ os.path.join(blob_dir, 'check_size.lock'))
+ except zc.lockfile.LockError:
+ # Someone is already cleaning up, so don't bother
+ return
+
+ try:
+ size = 0
+ blob_suffix = ZODB.blob.BLOB_SUFFIX
+ files_by_atime = BTrees.IOBTree.BTree()
+
+ for dirname in os.listdir(blob_dir):
+ if not cache_file_name(dirname):
+ continue
+ base = os.path.join(blob_dir, dirname)
+ if not os.path.isdir(base):
+ continue
+ for file_name in os.listdir(base):
+ if not file_name.endswith(blob_suffix):
+ continue
+ file_name = os.path.join(base, file_name)
+ if not os.path.isfile(file_name):
+ continue
+ stat = os.stat(file_name)
+ size += stat.st_size
+ t = stat.st_atime
+ if t not in files_by_atime:
+ files_by_atime[t] = []
+ files_by_atime[t].append(file_name)
+
+ while size > target and files_by_atime:
+ for file_name in files_by_atime.pop(files_by_atime.minKey()):
+ lockfilename = os.path.join(os.path.dirname(file_name),
+ '.lock')
+ try:
+ lock = zc.lockfile.LockFile(lockfilename)
+ except zc.lockfile.LockError:
+ continue # In use, skip
+
+ try:
+ size = os.stat(file_name).st_size
+ try:
+ ZODB.blob.remove_committed(file_name)
+ except OSError, v:
+ pass # probably open on windows
+ else:
+ size -= size
+ finally:
+ lock.close()
+ finally:
+ check_lock.close()
+
+def check_blob_size_script(args=None):
+ if args is None:
+ args = sys.argv[1:]
+ blob_dir, target = args
+ _check_blob_cache_size(blob_dir, int(target))
Modified: ZODB/branches/jim-dev/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/branches/jim-dev/src/ZEO/tests/testZEO.py 2008-12-04 18:51:29 UTC (rev 93629)
+++ ZODB/branches/jim-dev/src/ZEO/tests/testZEO.py 2008-12-04 19:46:22 UTC (rev 93630)
@@ -1171,7 +1171,7 @@
zeo.addTest(
doctest.DocFileSuite(
'zeo-fan-out.test', 'zdoptions.test',
- 'drop_cache_rather_than_verify.txt',
+ 'drop_cache_rather_than_verify.txt', 'client-config.test',
'protocols.test', 'zeo_blob_cache.test',
setUp=forker.setUp, tearDown=zope.testing.setupstack.tearDown,
),
Modified: ZODB/branches/jim-dev/src/ZODB/component.xml
===================================================================
--- ZODB/branches/jim-dev/src/ZODB/component.xml 2008-12-04 18:51:29 UTC (rev 93629)
+++ ZODB/branches/jim-dev/src/ZODB/component.xml 2008-12-04 19:46:22 UTC (rev 93630)
@@ -93,7 +93,24 @@
but only the filename when committing.
</description>
</key>
-
+ <key name="blob-cache-size" required="no" datatype="byte-size">
+ <description>
+ Maximum size of the ZEO blob cache, in bytes. If not set, then
+ the cache size isn't checked and the blob directory will
+ grow without bound.
+
+ This option is ignored if shared_blob_dir is true.
+ </description>
+ </key>
+ <key name="blob-cache-size-check" required="no" datatype="integer">
+ <description>
+ ZEO check size as percent of blob_cache_size. The ZEO
+ cache size will be checked when this many bytes have been
+ loaded into the cache. Defaults to 100% of the blob cache
+ size. This option is ignored if shared_blob_dir is true.
+ </description>
+ </key>
+
<key name="storage" default="1">
<description>
The name of the storage that the client wants to use. If the
Modified: ZODB/branches/jim-dev/src/ZODB/config.py
===================================================================
--- ZODB/branches/jim-dev/src/ZODB/config.py 2008-12-04 18:51:29 UTC (rev 93629)
+++ ZODB/branches/jim-dev/src/ZODB/config.py 2008-12-04 19:46:22 UTC (rev 93630)
@@ -164,6 +164,12 @@
# config.server is a multikey of socket-connection-address values
# where the value is a socket family, address tuple.
L = [server.address for server in self.config.server]
+ options = {}
+ if self.config.blob_cache_size is not None:
+ options['blob_cache_size'] = self.config.blob_cache_size
+ if self.config.blob_cache_size_check is not None:
+ options['blob_cache_size_check'] = self.config.blob_cache_size_check
+
return ClientStorage(
L,
blob_dir=self.config.blob_dir,
@@ -181,7 +187,8 @@
drop_cache_rather_verify=self.config.drop_cache_rather_verify,
username=self.config.username,
password=self.config.password,
- realm=self.config.realm)
+ realm=self.config.realm,
+ **options)
class BDBStorage(BaseConfig):
More information about the Zodb-checkins
mailing list