[Zodb-checkins] CVS: Zope/lib/python/ZEO - ClientCache.py:1.39.2.1 ClientStorage.py:1.76.2.1 ClientStub.py:1.10.4.1 CommitLog.py:1.4.8.1 Exceptions.py:1.5.8.1 ICache.py:1.3.8.1 README.txt:1.3.8.1 ServerStub.py:1.9.4.1 StorageServer.py:1.76.2.1 TransactionBuffer.py:1.8.4.1 __init__.py:1.13.2.1 simul.py:1.13.2.1 start.py:1.47.2.1 stats.py:1.18.2.1 util.py:1.4.2.1 version.txt:1.4.2.1
Chris McDonough
chrism@zope.com
Tue, 8 Oct 2002 20:41:45 -0400
Update of /cvs-repository/Zope/lib/python/ZEO
In directory cvs.zope.org:/tmp/cvs-serv15249/ZEO
Added Files:
Tag: chrism-install-branch
ClientCache.py ClientStorage.py ClientStub.py CommitLog.py
Exceptions.py ICache.py README.txt ServerStub.py
StorageServer.py TransactionBuffer.py __init__.py simul.py
start.py stats.py util.py version.txt
Log Message:
Committing ZEO to chrism-install-branch.
=== Added File Zope/lib/python/ZEO/ClientCache.py === (571/671 lines abridged)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# XXX TO DO
# use two indices rather than the sign bit of the index??????
# add a shared routine to read + verify a record???
# redesign header to include vdlen???
# rewrite the cache using a different algorithm???
"""Implement a client cache
The cache is managed as two files.
The cache can be persistent (meaning it is survives a process restart)
or temporary. It is persistent if the client argument is not None.
Persistent cache files live in the var directory and are named
'c<storage>-<client>-<digit>.zec' where <storage> is the storage
argument (default '1'), <client> is the client argument, and <digit> is
0 or 1. Temporary cache files are unnamed files in the standard
temporary directory as determined by the tempfile module.
The ClientStorage overrides the client name default to the value of
the environment variable ZEO_CLIENT, if it exists.
Each cache file has a 4-byte magic number followed by a sequence of
records of the form:
offset in record: name -- description
0: oid -- 8-byte object id
8: status -- 1-byte status 'v': valid, 'n': non-version valid, 'i': invalid
('n' means only the non-version data in the record is valid)
9: tlen -- 4-byte (unsigned) record length
13: vlen -- 2-byte (unsigned) version length
[-=- -=- -=- 571 lines omitted -=- -=- -=-]
if vlen+dlen+43+vdlen != tlen:
rilog("inconsistent lengths", pos, fileindex)
break
seek(vdlen, 1)
vs = read(8)
if read(4) != h[9:13]:
rilog("inconsistent tlen", pos, fileindex)
break
else:
if h[8] in 'vn' and vlen == 0:
if dlen+31 != tlen:
rilog("inconsistent nv lengths", pos, fileindex)
seek(dlen, 1)
if read(4) != h[9:13]:
rilog("inconsistent nv tlen", pos, fileindex)
break
vs = None
if h[8] in 'vn':
if fileindex:
index[oid] = -pos
else:
index[oid] = pos
serial[oid] = h[-8:], vs
else:
if serial.has_key(oid):
# We have a record for this oid, but it was invalidated!
del serial[oid]
del index[oid]
pos = pos + tlen
count += 1
f.seek(pos)
try:
f.truncate()
except:
pass
if count:
log("read_index: cache file %d has %d records and %d bytes"
% (fileindex, count, pos))
return pos
def rilog(msg, pos, fileindex):
# Helper to log messages from read_index
log("read_index: %s at position %d in cache file %d"
% (msg, pos, fileindex))
=== Added File Zope/lib/python/ZEO/ClientStorage.py === (694/794 lines abridged)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The ClientStorage class and the exceptions that it may raise.
Public contents of this module:
ClientStorage -- the main class, implementing the Storage API
ClientStorageError -- exception raised by ClientStorage
UnrecognizedResult -- exception raised by ClientStorage
ClientDisconnected -- exception raised by ClientStorage
"""
# XXX TO DO
# get rid of beginVerify, set up _tfile in verify_cache
# set self._storage = stub later, in endVerify
# if wait is given, wait until verify is complete
import cPickle
import os
import tempfile
import threading
import time
from ZEO import ClientCache, ServerStub
from ZEO.TransactionBuffer import TransactionBuffer
from ZEO.Exceptions import Disconnected
from ZEO.zrpc.client import ConnectionManager
from ZODB import POSException
from ZODB.TimeStamp import TimeStamp
from zLOG import LOG, PROBLEM, INFO, BLATHER
def log2(type, msg, subsys="ZCS:%d" % os.getpid()):
LOG(subsys, type, msg)
try:
from ZODB.ConflictResolution import ResolvedSerial
except ImportError:
ResolvedSerial = 'rs'
[-=- -=- -=- 694 lines omitted -=- -=- -=-]
# Queue an invalidate for the end the verification procedure.
if self._pickler is None:
# XXX This should never happen
return
self._pickler.dump(args)
def endVerify(self):
"""Server callback to signal end of cache validation."""
if self._pickler is None:
return
self._pickler.dump((0,0))
self._tfile.seek(0)
unpick = cPickle.Unpickler(self._tfile)
f = self._tfile
self._tfile = None
while 1:
oid, version = unpick.load()
if not oid:
break
self._cache.invalidate(oid, version=version)
self._db.invalidate(oid, version=version)
f.close()
def invalidateTrans(self, args):
"""Server callback to invalidate a list of (oid, version) pairs.
This is called as the result of a transaction.
"""
for oid, version in args:
self._cache.invalidate(oid, version=version)
try:
self._db.invalidate(oid, version=version)
except AttributeError, msg:
log2(PROBLEM,
"Invalidate(%s, %s) failed for _db: %s" % (repr(oid),
repr(version),
msg))
# Unfortunately, the ZEO 2 wire protocol uses different names for
# several of the callback methods invoked by the StorageServer.
# We can't change the wire protocol at this point because that
# would require synchronized updates of clients and servers and we
# don't want that. So here we alias the old names to their new
# implementations.
begin = beginVerify
invalidate = invalidateVerify
end = endVerify
Invalidate = invalidateTrans
=== Added File Zope/lib/python/ZEO/ClientStub.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""RPC stubs for interface exported by ClientStorage."""
class ClientStorage:
"""An RPC stub class for the interface exported by ClientStorage.
This is the interface presented by ClientStorage to the
StorageServer; i.e. the StorageServer calls these methods and they
are executed in the ClientStorage.
See the ClientStorage class for documentation on these methods.
It is currently important that all methods here are asynchronous
(meaning they don't have a return value and the caller doesn't
wait for them to complete), *and* that none of them cause any
calls from the client to the storage. This is due to limitations
in the zrpc subpackage.
The on-the-wire names of some of the methods don't match the
Python method names. That's because the on-the-wire protocol was
fixed for ZEO 2 and we don't want to change it. There are some
aliases in ClientStorage.py to make up for this.
"""
def __init__(self, rpc):
"""Constructor.
The argument is a connection: an instance of the
zrpc.connection.Connection class.
"""
self.rpc = rpc
def beginVerify(self):
self.rpc.callAsync('begin')
def invalidateVerify(self, args):
self.rpc.callAsync('invalidate', args)
def endVerify(self):
self.rpc.callAsync('end')
def invalidateTrans(self, args):
self.rpc.callAsync('Invalidate', args)
def serialnos(self, arg):
self.rpc.callAsync('serialnos', arg)
def info(self, arg):
self.rpc.callAsync('info', arg)
=== Added File Zope/lib/python/ZEO/CommitLog.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Log a transaction's commit info during two-phase commit.
A storage server allows multiple clients to commit transactions, but
must serialize them as the actually execute at the server. The
concurrent commits are achieved by logging actions up until the
tpc_vote(). At that point, the entire transaction is committed on the
real storage.
"""
import cPickle
import tempfile
class CommitLog:
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".log")
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
self.stores = 0
self.read = 0
def store(self, oid, serial, data, version):
self.pickler.dump((oid, serial, data, version))
self.stores += 1
def get_loader(self):
self.read = 1
self.file.seek(0)
return self.stores, cPickle.Unpickler(self.file)
=== Added File Zope/lib/python/ZEO/Exceptions.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Exceptions for ZEO."""
class Disconnected(Exception):
"""Exception raised when a ZEO client is disconnected from the
ZEO server."""
=== Added File Zope/lib/python/ZEO/ICache.py ===
try:
from Interface import Base
except ImportError:
class Base:
# a dummy interface for use when Zope's is unavailable
pass
class ICache(Base):
"""ZEO client cache.
__init__(storage, size, client, var)
All arguments optional.
storage -- name of storage
size -- max size of cache in bytes
client -- a string; if specified, cache is persistent.
var -- var directory to store cache files in
"""
def open():
"""Returns a sequence of object info tuples.
An object info tuple is a pair containing an object id and a
pair of serialnos, a non-version serialno and a version serialno:
oid, (serial, ver_serial)
This method builds an index of the cache and returns a
sequence used for cache validation.
"""
def close():
"""Closes the cache."""
def verify(func):
"""Call func on every object in cache.
func is called with three arguments
func(oid, serial, ver_serial)
"""
def invalidate(oid, version):
"""Remove object from cache."""
def load(oid, version):
"""Load object from cache.
Return None if object not in cache.
Return data, serialno if object is in cache.
"""
def store(oid, p, s, version, pv, sv):
"""Store a new object in the cache."""
def update(oid, serial, version, data):
"""Update an object already in the cache.
XXX This method is called to update objects that were modified by
a transaction. It's likely that it is already in the cache,
and it may be possible for the implementation to operate more
efficiently.
"""
def modifiedInVersion(oid):
"""Return the version an object is modified in.
'' signifies the trunk.
Returns None if the object is not in the cache.
"""
def checkSize(size):
"""Check if adding size bytes would exceed cache limit.
This method is often called just before store or update. The
size is a hint about the amount of data that is about to be
stored. The cache may want to evict some data to make space.
"""
=== Added File Zope/lib/python/ZEO/README.txt ===
=======
ZEO 2.0
=======
What's ZEO?
===========
ZEO stands for Zope Enterprise Objects. ZEO is an add-on for Zope
that allows a multiple processes to connect to a single ZODB storage.
Those processes can live on different machines, but don't need to.
ZEO 2 has many improvements over ZEO 1, and is incompatible with ZEO
1; if you upgrade an existing ZEO 1 installation, you must upgrade the
server and all clients simultaneous. If you received ZEO 2 as part of
the ZODB 3 distribution, the ZEO 1 sources are provided in a separate
directory (ZEO1). Some documentation for ZEO is available in the ZODB
3 package in the Doc subdirectory. ZEO depends on the ZODB software;
it can be used with the version of ZODB distributed with Zope 2.5.1 or
later. More information about ZEO can be found on its home on the
web:
http://www.zope.org/Products/ZEO/
What's here?
============
This list of filenames is mostly for ZEO developers::
ClientCache.py client-side cache implementation
ClientStorage.py client-side storage implementation
ClientStub.py RPC stubs for callbacks from server to client
CommitLog.py buffer used during two-phase commit on the server
Exceptions.py definitions of exceptions
ICache.py interface definition for the client-side cache
ServerStub.py RPC stubs for the server
StorageServer.py server-side storage implementation
TransactionBuffer.py buffer used for transaction data in the client
__init__.py near-empty file to make this directory a package
simul.py command-line tool to simulate cache behavior
start.py command-line tool to start the storage server
stats.py command-line tool to process client cache traces
tests/ unit tests and other test utilities
util.py utilities used by the server startup tool
version.txt text file indicating the ZEO version
zrpc/ subpackage implementing Remote Procedure Call (RPC)
Client Cache Tracing
====================
An important question for ZEO users is: how large should the ZEO
client cache be? ZEO 2 (as of ZEO 2.0b2) has a new feature that lets
you collect a trace of cache activity and tools to analyze this trace,
enabling you to make an informed decision about the cache size.
Don't confuse the ZEO client cache with the Zope object cache. The
ZEO client cache is only used when an object is not in the Zope object
cache; the ZEO client cache avoids roundtrips to the ZEO server.
Enabling Cache Tracing
----------------------
To enable cache tracing, set the environment variable ZEO_CACHE_TRACE
to the name of a file to which the ZEO client process can write. If
the file doesn't exist, the ZEO will try to create it. If there are
problems with the file, a log message is written to the standard Zope
log file. To start or stop tracing, the ZEO client process (typically
a Zope application server) must be restarted.
The trace file can grow pretty quickly; on a moderately loaded server,
we observed it growing by 5 MB per hour. The file consists of binary
records, each 24 bytes long; a detailed description of the record
lay-out is given in stats.py. No sensitive data is logged.
Analyzing a Cache Trace
-----------------------
The stats.py command-line tool is the first-line tool to analyze a
cache trace. Its default output consists of two parts: a one-line
summary of essential statistics for each segment of 15 minutes,
interspersed with lines indicating client restarts and "cache flip
events" (more about those later), followed by a more detailed summary
of overall statistics.
The most important statistic is probably the "hit rate", a percentage
indicating how many requests to load an object could be satisfied from
the cache. Hit rates around 70% are good. 90% is probably close to
the theoretical maximum. If you see a hit rate under 60% you can
probably improve the cache performance (and hence your Zope
application server's performance) by increasing the ZEO cache size.
This is normally configured using the cache_size keyword argument to
the ClientStorage() constructor in your custom_zodb.py file. The
default cache size is 20 MB.
The stats.py tool shows its command line syntax when invoked without
arguments. The tracefile argument can be a gzipped file if it has a
.gz extension. It will read from stdin (assuming uncompressed data)
if the tracefile argument is '-'.
Simulating Different Cache Sizes
--------------------------------
Based on a cache trace file, you can make a prediction of how well the
cache might do with a different cache size. The simul.py tool runs an
accurate simulation of the ZEO client cache implementation based upon
the events read from a trace file. A new simulation is started each
time the trace file records a client restart event; if a trace file
contains more than one restart event, a separate line is printed for
each simulation, and line with overall statistics is added at the end.
Example, assuming the trace file is in /tmp/cachetrace.log::
$ python simul.py -s 100 /tmp/cachetrace.log
START TIME DURATION LOADS HITS INVALS WRITES FLIPS HITRATE
Sep 4 11:59 38:01 59833 40473 257 20 2 67.6%
$
This shows that with a 100 MB cache size, the cache hit rate is
67.6%. So let's try this again with a 200 MB cache size::
$ python simul.py -s 200 /tmp/cachetrace.log
START TIME DURATION LOADS HITS INVALS WRITES FLIPS HITRATE
Sep 4 11:59 38:01 59833 40921 258 20 1 68.4%
$
This showed hardly any improvement. So let's try a 300 MB cache
size::
$ python2.0 simul.py -s 300 /tmp/cachetrace.log
ZEOCacheSimulation, cache size 300,000,000 bytes
START TIME DURATION LOADS HITS INVALS WRITES FLIPS HITRATE
Sep 4 11:59 38:01 59833 40921 258 20 0 68.4%
$
This shows that for this particular trace file, the maximum attainable
hit rate is 68.4%. This is probably caused by the fact that nearly a
third of the objects mentioned in the trace were loaded only once --
the cache only helps if an object is loaded more than once.
The simul.py tool also supports simulating different cache
strategies. Since none of these are implemented, these are not
further documented here.
Cache Flips
-----------
The cache uses two files, which are managed as follows:
- Data are written to file 0 until file 0 exceeds limit/2 in size.
- Data are written to file 1 until file 1 exceeds limit/2 in size.
- File 0 is truncated to size 0 (or deleted and recreated).
- Data are written to file 0 until file 0 exceeds limit/2 in size.
- File 1 is truncated to size 0 (or deleted and recreated).
- Data are written to file 1 until file 1 exceeds limit/2 in size.
and so on.
A switch from file 0 to file 1 is called a "cache flip". At all cache
flips except the first, half of the cache contents is wiped out. This
affects cache performance. How badly this impact is can be seen from
the per-15-minutes summaries printed by stats.py. The -i option lets
you choose a smaller summary interval which shows the impact more
acutely.
The simul.py tool shows the number of cache flips in the FLIPS column.
If you see more than one flip per hour the cache may be too small.
=== Added File Zope/lib/python/ZEO/ServerStub.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""RPC stubs for interface exported by StorageServer."""
class StorageServer:
"""An RPC stub class for the interface exported by ClientStorage.
This is the interface presented by the StorageServer to the
ClientStorage; i.e. the ClientStorage calls these methods and they
are executed in the StorageServer.
See the StorageServer module for documentation on these methods,
with the exception of _update(), which is documented here.
"""
def __init__(self, rpc):
"""Constructor.
The argument is a connection: an instance of the
zrpc.connection.Connection class.
"""
self.rpc = rpc
def _update(self):
"""Handle pending incoming messages.
This method is typically only used when no asyncore mainloop
is already active. It can cause arbitrary callbacks from the
server to the client to be handled.
"""
self.rpc.pending()
def register(self, storage_name, read_only):
self.rpc.call('register', storage_name, read_only)
def get_info(self):
return self.rpc.call('get_info')
def beginZeoVerify(self):
self.rpc.callAsync('beginZeoVerify')
def zeoVerify(self, oid, s, sv):
self.rpc.callAsync('zeoVerify', oid, s, sv)
def endZeoVerify(self):
self.rpc.callAsync('endZeoVerify')
def new_oids(self, n=None):
if n is None:
return self.rpc.call('new_oids')
else:
return self.rpc.call('new_oids', n)
def pack(self, t, wait=None):
if wait is None:
self.rpc.call('pack', t)
else:
self.rpc.call('pack', t, wait)
def zeoLoad(self, oid):
return self.rpc.call('zeoLoad', oid)
def storea(self, oid, serial, data, version, id):
self.rpc.callAsync('storea', oid, serial, data, version, id)
def tpc_begin(self, id, user, descr, ext, tid, status):
return self.rpc.call('tpc_begin', id, user, descr, ext, tid, status)
def vote(self, trans_id):
return self.rpc.call('vote', trans_id)
def tpc_finish(self, id):
return self.rpc.call('tpc_finish', id)
def tpc_abort(self, id):
self.rpc.callAsync('tpc_abort', id)
def abortVersion(self, src, id):
return self.rpc.call('abortVersion', src, id)
def commitVersion(self, src, dest, id):
return self.rpc.call('commitVersion', src, dest, id)
def history(self, oid, version, length=None):
if length is None:
return self.rpc.call('history', oid, version)
else:
return self.rpc.call('history', oid, version, length)
def load(self, oid, version):
return self.rpc.call('load', oid, version)
def loadSerial(self, oid, serial):
return self.rpc.call('loadSerial', oid, serial)
def modifiedInVersion(self, oid):
return self.rpc.call('modifiedInVersion', oid)
def new_oid(self, last=None):
if last is None:
return self.rpc.call('new_oid')
else:
return self.rpc.call('new_oid', last)
def store(self, oid, serial, data, version, trans):
return self.rpc.call('store', oid, serial, data, version, trans)
def transactionalUndo(self, trans_id, trans):
return self.rpc.call('transactionalUndo', trans_id, trans)
def undo(self, trans_id):
return self.rpc.call('undo', trans_id)
def undoLog(self, first, last):
return self.rpc.call('undoLog', first, last)
def undoInfo(self, first, last, spec):
return self.rpc.call('undoInfo', first, last, spec)
def versionEmpty(self, vers):
return self.rpc.call('versionEmpty', vers)
def versions(self, max=None):
if max is None:
return self.rpc.call('versions')
else:
return self.rpc.call('versions', max)
=== Added File Zope/lib/python/ZEO/StorageServer.py === (643/743 lines abridged)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The StorageServer class and the exception that it may raise.
This server acts as a front-end for one or more real storages, like
file storage or Berkeley storage.
XXX Need some basic access control-- a declaration of the methods
exported for invocation by the server.
"""
import asyncore
import cPickle
import os
import sys
import threading
from ZEO import ClientStub
from ZEO.CommitLog import CommitLog
from ZEO.zrpc.server import Dispatcher
from ZEO.zrpc.connection import ManagedServerConnection, Delay, MTDelay
import zLOG
from ZODB.POSException import StorageError, StorageTransactionError
from ZODB.POSException import TransactionError, ReadOnlyError
from ZODB.referencesf import referencesf
from ZODB.Transaction import Transaction
_label = "ZSS" # Default label used for logging.
def set_label():
"""Internal helper to reset the logging label (e.g. after fork())."""
global _label
_label = "ZSS:%s" % os.getpid()
def log(message, level=zLOG.INFO, label=None, error=None):
"""Internal helper to log a message using zLOG."""
zLOG.LOG(label or _label, level, message, error=error)
[-=- -=- -=- 643 lines omitted -=- -=- -=-]
oid, serial, data, version = loader.load()
new_strategy.store(oid, serial, data, version)
meth = getattr(new_strategy, self.name)
return meth(*self.args)
def abort(self, zeo_storage):
# Delete (d, zeo_storage) from the _waiting list, if found.
waiting = self.storage._waiting
for i in range(len(waiting)):
d, z = waiting[i]
if z is zeo_storage:
del waiting[i]
break
def run_in_thread(method, *args):
t = SlowMethodThread(method, args)
t.start()
return t.delay
class SlowMethodThread(threading.Thread):
"""Thread to run potentially slow storage methods.
Clients can use the delay attribute to access the MTDelay object
used to send a zrpc response at the right time.
"""
# Some storage methods can take a long time to complete. If we
# run these methods via a standard asyncore read handler, they
# will block all other server activity until they complete. To
# avoid blocking, we spawn a separate thread, return an MTDelay()
# object, and have the thread reply() when it finishes.
def __init__(self, method, args):
threading.Thread.__init__(self)
self._method = method
self._args = args
self.delay = MTDelay()
def run(self):
try:
result = self._method(*self._args)
except Exception:
self.delay.error(sys.exc_info())
else:
self.delay.reply(result)
# Patch up class references
StorageServer.ZEOStorageClass = ZEOStorage
ZEOStorage.DelayedCommitStrategyClass = DelayedCommitStrategy
ZEOStorage.ImmediateCommitStrategyClass = ImmediateCommitStrategy
=== Added File Zope/lib/python/ZEO/TransactionBuffer.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A TransactionBuffer store transaction updates until commit or abort.
A transaction may generate enough data that it is not practical to
always hold pending updates in memory. Instead, a TransactionBuffer
is used to store the data until a commit or abort.
"""
# A faster implementation might store trans data in memory until it
# reaches a certain size.
import cPickle
import tempfile
from threading import Lock
class TransactionBuffer:
# Valid call sequences:
#
# ((store | invalidate)* begin_iterate next* clear)* close
#
# get_size can be called any time
# The TransactionBuffer is used by client storage to hold update
# data until the tpc_finish(). It is normally used by a single
# thread, because only one thread can be in the two-phase commit
# at one time.
# It is possible, however, for one thread to close the storage
# while another thread is in the two-phase commit. We must use
# a lock to guard against this race, because unpredictable things
# can happen in Python if one thread closes a file that another
# thread is reading. In a debug build, an assert() can fail.
# XXX If an operation is performed on a closed TransactionBuffer,
# it has no effect and does not raise an exception. The only time
# this should occur is when a ClientStorage is closed in one
# thread while another thread is in its tpc_finish(). It's not
# clear what should happen in this case. If the tpc_finish()
# completes without error, the Connection using it could have
# inconsistent data. This should have minimal effect, though,
# because the Connection is connected to a closed storage.
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".tbuf")
self.lock = Lock()
self.closed = 0
self.count = 0
self.size = 0
# It's safe to use a fast pickler because the only objects
# stored are builtin types -- strings or None.
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
def close(self):
self.lock.acquire()
try:
self.closed = 1
try:
self.file.close()
except OSError:
pass
finally:
self.lock.release()
def store(self, oid, version, data):
self.lock.acquire()
try:
self._store(oid, version, data)
finally:
self.lock.release()
def _store(self, oid, version, data):
"""Store oid, version, data for later retrieval"""
if self.closed:
return
self.pickler.dump((oid, version, data))
self.count += 1
# Estimate per-record cache size
self.size = self.size + len(data) + 31
if version:
# Assume version data has same size as non-version data
self.size = self.size + len(version) + len(data) + 12
def invalidate(self, oid, version):
self.lock.acquire()
try:
if self.closed:
return
self.pickler.dump((oid, version, None))
self.count += 1
finally:
self.lock.release()
def clear(self):
"""Mark the buffer as empty"""
self.lock.acquire()
try:
if self.closed:
return
self.file.seek(0)
self.count = 0
self.size = 0
finally:
self.lock.release()
# unchecked constraints:
# 1. can't call store() after begin_iterate()
# 2. must call clear() after iteration finishes
def begin_iterate(self):
"""Move the file pointer in advance of iteration"""
self.lock.acquire()
try:
if self.closed:
return
self.file.flush()
self.file.seek(0)
self.unpickler = cPickle.Unpickler(self.file)
finally:
self.lock.release()
def next(self):
self.lock.acquire()
try:
return self._next()
finally:
self.lock.release()
def _next(self):
"""Return next tuple of data or None if EOF"""
if self.closed:
return None
if self.count == 0:
del self.unpickler
return None
oid_ver_data = self.unpickler.load()
self.count -= 1
return oid_ver_data
def get_size(self):
"""Return size of data stored in buffer (just a hint)."""
return self.size
=== Added File Zope/lib/python/ZEO/__init__.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZEO -- Zope Enterprise Objects.
See the file README.txt in this directory for an overview.
ZEO's home on the web is
http://www.zope.org/Products/ZEO/
"""
version = "2.0b2"
=== Added File Zope/lib/python/ZEO/simul.py === (638/738 lines abridged)
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Cache simulation.
Usage: simul.py [-bflyz] [-s size] tracefile
Use one of -b, -f, -l, -y or -z select the cache simulator:
-b: buddy system allocator
-f: simple free list allocator
-l: idealized LRU (no allocator)
-y: variation on the existing ZEO cache that copies to current file
-z: existing ZEO cache (default)
Options:
-s size: cache size in MB (default 20 MB)
Note: the buddy system allocator rounds the cache size up to a power of 2
"""
import sys
import time
import getopt
import struct
def usage(msg):
print >>sys.stderr, msg
print >>sys.stderr, __doc__
def main():
# Parse options
MB = 1000*1000
cachelimit = 20*MB
simclass = ZEOCacheSimulation
try:
opts, args = getopt.getopt(sys.argv[1:], "bflyzs:")
except getopt.error, msg:
usage(msg)
return 2
[-=- -=- -=- 638 lines omitted -=- -=- -=-]
queue = []
T = 0
blocks = 0
while T < 5000:
while queue and queue[0][0] <= T:
time, node = heapq.heappop(queue)
assert time == T
##print "free addr=%d, size=%d" % (node.addr, node.size)
cache.free(node)
blocks -= 1
size = random.randint(100, 2000)
lifetime = random.randint(1, 100)
node = cache.alloc(size)
if node is None:
print "out of mem"
cache.dump("T=%4d: %d blocks;" % (T, blocks))
break
else:
##print "alloc addr=%d, size=%d" % (node.addr, node.size)
blocks += 1
heapq.heappush(queue, (T + lifetime, node))
T = T+1
if T % reportfreq == 0:
cache.dump("T=%4d: %d blocks;" % (T, blocks))
def hitrate(loads, hits):
return "%5.1f%%" % (100.0 * hits / max(1, loads))
def duration(secs):
mm, ss = divmod(secs, 60)
hh, mm = divmod(mm, 60)
if hh:
return "%d:%02d:%02d" % (hh, mm, ss)
if mm:
return "%d:%02d" % (mm, ss)
return "%d" % ss
def addcommas(n):
sign, s = '', str(n)
if s[0] == '-':
sign, s = '-', s[1:]
i = len(s) - 3
while i > 0:
s = s[:i] + ',' + s[i:]
i -= 3
return sign + s
if __name__ == "__main__":
sys.exit(main())
=== Added File Zope/lib/python/ZEO/start.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Start the ZEO storage server."""
from __future__ import nested_scopes
import sys, os, getopt
import types
def directory(p, n=1):
d = p
while n:
d = os.path.split(d)[0]
if not d or d == '.':
d = os.getcwd()
n -= 1
return d
def get_storage(m, n, cache={}):
p = sys.path
d, m = os.path.split(m)
if m.endswith('.py'):
m = m[:-3]
im = cache.get((d, m))
if im is None:
if d:
p = [d] + p
import imp
im = imp.find_module(m, p)
im = imp.load_module(m, *im)
cache[(d, m)] = im
return getattr(im, n)
def set_uid(arg):
"""Try to set uid and gid based on -u argument.
This will only work if this script is run by root.
"""
try:
import pwd
except ImportError:
LOG('ZEO/start.py', INFO, ("Can't set uid to %s."
"pwd module is not available." % arg))
return
try:
gid = None
try:
arg = int(arg)
except: # conversion could raise all sorts of errors
uid = pwd.getpwnam(arg)[2]
gid = pwd.getpwnam(arg)[3]
else:
uid = pwd.getpwuid(arg)[2]
gid = pwd.getpwuid(arg)[3]
if gid is not None:
try:
os.setgid(gid)
except OSError:
pass
try:
os.setuid(uid)
except OSError:
pass
except KeyError:
LOG('ZEO/start.py', ERROR, ("can't find uid %s" % arg))
def setup_signals(storages):
try:
import signal
except ImportError:
return
try:
xfsz = signal.SIFXFSZ
except AttributeError:
pass
else:
signal.signal(xfsz, signal.SIG_IGN)
signal.signal(signal.SIGTERM, lambda sig, frame: shutdown(storages))
signal.signal(signal.SIGINT, lambda sig, frame: shutdown(storages, 0))
try:
signal.signal(signal.SIGHUP, rotate_logs_handler)
except:
pass
def main(argv):
me = argv[0]
sys.path.insert(0, directory(me, 2))
global LOG, INFO, ERROR
from zLOG import LOG, INFO, ERROR, PANIC
from ZEO.util import Environment
env = Environment(me)
# XXX hack for profiling support
global unix, storages, asyncore
args = []
last = ''
for a in argv[1:]:
if (a[:1] != '-' and a.find('=') > 0 and last != '-S'): # lame, sorry
a = a.split("=")
os.environ[a[0]] = "=".join(a[1:])
continue
args.append(a)
last = a
usage="""%s [options] [filename]
where options are:
-D -- Run in debug mode
-d -- Set STUPD_LOG_SEVERITY to -300
-U -- Unix-domain socket file to listen on
-u username or uid number
The username to run the ZEO server as. You may want to run
the ZEO server as 'nobody' or some other user with limited
resouces. The only works under Unix, and if ZServer is
started by root.
-p port -- port to listen on
-h adddress -- host address to listen on
-s -- Don't use zdeamon
-S storage_name=module_path:attr_name -- A storage specification
where:
storage_name -- is the storage name used in the ZEO protocol.
This is the name that you give as the optional
'storage' keyword argument to the ClientStorage constructor.
module_path -- This is the path to a Python module
that defines the storage object(s) to be served.
The module path should omit the prefix (e.g. '.py').
attr_name -- This is the name to which the storage object
is assigned in the module.
-P file -- Run under profile and dump output to file. Implies the
-s flag.
if no file name is specified, then %s is used.
""" % (me, env.fs)
try:
opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:P:d')
except getopt.error, msg:
print usage
print msg
sys.exit(1)
port = None
debug = 0
host = ''
unix = None
Z = 1
UID = 'nobody'
prof = None
detailed = 0
fs = None
for o, v in opts:
if o =='-p':
port = int(v)
elif o =='-h':
host = v
elif o =='-U':
unix = v
elif o =='-u':
UID = v
elif o =='-D':
debug = 1
elif o =='-d':
detailed = 1
elif o =='-s':
Z = 0
elif o =='-P':
prof = v
if prof:
Z = 0
if port is None and unix is None:
print usage
print 'No port specified.'
sys.exit(1)
if args:
if len(args) > 1:
print usage
print 'Unrecognizd arguments: ', " ".join(args[1:])
sys.exit(1)
fs = args[0]
if debug:
os.environ['Z_DEBUG_MODE'] = '1'
if detailed:
os.environ['STUPID_LOG_SEVERITY'] = '-300'
set_uid(UID)
if Z:
try:
import posix
except:
pass
else:
import zdaemon
zdaemon.run(sys.argv, '')
try:
import ZEO.StorageServer, asyncore
storages = {}
for o, v in opts:
if o == '-S':
n, m = v.split("=", 1)
if m.find(":") >= 0:
# we got an attribute name
m, a = m.split(':')
else:
# attribute name must be same as storage name
a=n
storages[n]=get_storage(m,a)
if not storages:
from ZODB.FileStorage import FileStorage
storages['1'] = FileStorage(fs or env.fs)
# Try to set up a signal handler
setup_signals(storages)
items = storages.items()
items.sort()
for kv in items:
LOG('ZEO/start.py', INFO, 'Serving %s:\t%s' % kv)
if not unix:
unix = host, port
ZEO.StorageServer.StorageServer(unix, storages)
try:
ppid, pid = os.getppid(), os.getpid()
except:
pass # getpid not supported
else:
f = open(env.zeo_pid, 'w')
f.write("%s %s\n" % (ppid, pid))
f.close()
except:
# Log startup exception and tell zdaemon not to restart us.
info = sys.exc_info()
try:
LOG("ZEO/start.py", PANIC, "Startup exception", error=info)
except:
pass
import traceback
traceback.print_exception(*info)
sys.exit(0)
try:
try:
asyncore.loop()
finally:
if os.path.isfile(env.zeo_pid):
os.unlink(env.zeo_pid)
except SystemExit:
raise
except:
info = sys.exc_info()
try:
LOG("ZEO/start.py", PANIC, "Unexpected error", error=info)
except:
pass
import traceback
traceback.print_exception(*info)
sys.exit(1)
def rotate_logs():
import zLOG
# There hasn't been a clear way to reinitialize the MinimalLogger.
# I'll checkin the public initialize() method soon, but also try some
# other strategies for older Zope installs :-(.
init = getattr(zLOG, 'initialize', None)
if init is not None:
init()
return
# This will work if the minimal logger is in use, but not if some
# other logger is active.
import zLOG.MinimalLogger
zLOG.MinimalLogger._log.initialize()
def rotate_logs_handler(signum, frame):
rotate_logs()
import signal
signal.signal(signal.SIGHUP, rotate_logs_handler)
def shutdown(storages, die=1):
LOG("ZEO/start.py", INFO, "Received signal")
import asyncore
# Do this twice, in case we got some more connections
# while going through the loop. This is really sort of
# unnecessary, since we now use so_reuseaddr.
for ignored in 1,2:
for socket in asyncore.socket_map.values():
try:
socket.close()
except:
pass
for storage in storages.values():
try:
storage.close()
finally:
pass
try:
s = die and "shutdown" or "restart"
LOG('ZEO/start.py', INFO, "Shutting down (%s)" % s)
except:
pass
if die:
sys.exit(0)
else:
sys.exit(1)
if __name__=='__main__':
main(sys.argv)
=== Added File Zope/lib/python/ZEO/stats.py ===
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Trace file statistics analyzer.
Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] tracefile
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
"""
"""File format:
Each record is 24 bytes, with the following layout. Numbers are
big-endian integers.
Offset Size Contents
0 4 timestamp (seconds since 1/1/1970)
4 3 data size, in 256-byte increments, rounded up
7 1 code (see below)
8 8 object id
16 8 serial number
The code at offset 7 packs three fields:
Mask bits Contents
0x80 1 set if there was a non-empty version string
0x7e 6 function and outcome code
0x01 1 current cache file (0 or 1)
The function and outcome codes are documented in detail at the end of
this file in the 'explain' dictionary. Note that the keys there (and
also the arguments to _trace() in ClientStorage.py) are 'code & 0x7e',
i.e. the low bit is always zero.
"""
import sys
import time
import getopt
import struct
def usage(msg):
print >>sys.stderr, msg
print >>sys.stderr, __doc__
def main():
# Parse options
verbose = 0
quiet = 0
dostats = 1
print_size_histogram = 0
print_histogram = 0
interval = 900 # Every 15 minutes
try:
opts, args = getopt.getopt(sys.argv[1:], "hi:qsSv")
except getopt.error, msg:
usage(msg)
return 2
for o, a in opts:
if o == '-h':
print_histogram = 1
if o == "-i":
interval = int(60 * float(a))
if interval <= 0:
interval = 60
elif interval > 3600:
interval = 3600
if o == "-q":
quiet = 1
verbose = 0
if o == "-s":
print_size_histogram = 1
if o == "-S":
dostats = 0
if o == "-v":
verbose = 1
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file
if filename.endswith(".gz"):
# Open gzipped file
try:
import gzip
except ImportError:
print >>sys.stderr, "can't read gzipped files (no module gzip)"
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >>sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
elif filename == '-':
# Read from stdin
f = sys.stdin
else:
# Open regular file
try:
f = open(filename, "rb")
except IOError, msg:
print >>sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
# Read file, gathering statistics, and printing each record if verbose
rt0 = time.time()
bycode = {}
records = 0
versions = 0
t0 = te = None
datarecords = 0
datasize = 0L
file0 = file1 = 0
oids = {}
bysize = {}
bysizew = {}
total_loads = 0
byinterval = {}
thisinterval = None
h0 = he = None
offset = 0
f_read = f.read
struct_unpack = struct.unpack
try:
while 1:
r = f_read(8)
if len(r) < 8:
break
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
if not quiet:
print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(16)
if len(r) < 16:
break
offset += 16
records += 1
oid, serial = struct_unpack(">8s8s", r)
if t0 is None:
t0 = ts
thisinterval = t0 / interval
h0 = he = ts
te = ts
if ts / interval != thisinterval:
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts / interval
h0 = ts
he = ts
dlen, code = code & 0x7fffff00, code & 0xff
if dlen:
datarecords += 1
datasize += dlen
version = '-'
if code & 0x80:
version = 'V'
versions += 1
current = code & 1
if current:
file1 += 1
else:
file0 += 1
code = code & 0x7e
bycode[code] = bycode.get(code, 0) + 1
byinterval[code] = byinterval.get(code, 0) + 1
if dlen:
if code & 0x70 == 0x20: # All loads
bysize[dlen] = d = bysize.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
elif code == 0x3A: # Update
bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
if verbose:
print "%s %d %02x %016x %016x %1s %s" % (
time.ctime(ts)[4:-5],
current,
code,
U64(oid),
U64(serial),
version,
dlen and str(dlen) or "")
if code & 0x70 == 0x20:
oids[oid] = oids.get(oid, 0) + 1
total_loads += 1
if code in (0x00, 0x70):
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts / interval
h0 = he = ts
if not quiet:
print time.ctime(ts)[4:-5],
if code == 0x00:
print '='*20, "Restart", '='*20
else:
print '-'*20, "Flip->%d" % current, '-'*20
except KeyboardInterrupt:
print "\nInterrupted. Stats so far:\n"
f.close()
rte = time.time()
if not quiet:
dumpbyinterval(byinterval, h0, he)
# Error if nothing was read
if not records:
print >>sys.stderr, "No records processed"
return 1
# Print statistics
if dostats:
print
print "Read %s records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(records*24), rte-rt0)
print "Versions: %s records used a version" % addcommas(versions)
print "First time: %s" % time.ctime(t0)
print "Last time: %s" % time.ctime(te)
print "Duration: %s seconds" % addcommas(te-t0)
print "File stats: %s in file 0; %s in file 1" % (
addcommas(file0), addcommas(file1))
print "Data recs: %s (%.1f%%), average size %.1f KB" % (
addcommas(datarecords),
100.0 * datarecords / records,
datasize / 1024.0 / datarecords)
print "Hit rate: %.1f%% (load hits / loads)" % hitrate(bycode)
print
codes = bycode.keys()
codes.sort()
print "%13s %4s %s" % ("Count", "Code", "Function (action)")
for code in codes:
print "%13s %02x %s" % (
addcommas(bycode.get(code, 0)),
code,
explain.get(code) or "*** unknown code ***")
# Print histogram
if print_histogram:
print
print "Histogram of object load frequency"
total = len(oids)
print "Unique oids: %s" % addcommas(total)
print "Total loads: %s" % addcommas(total_loads)
s = addcommas(total)
width = max(len(s), len("objects"))
fmt = "%5d %" + str(width) + "s %5.1f%% %5.1f%% %5.1f%%"
hdr = "%5s %" + str(width) + "s %6s %6s %6s"
print hdr % ("loads", "objects", "%obj", "%load", "%cum")
cum = 0.0
for binsize, count in histogram(oids):
obj_percent = 100.0 * count / total
load_percent = 100.0 * count * binsize / total_loads
cum += load_percent
print fmt % (binsize, addcommas(count),
obj_percent, load_percent, cum)
# Print size histogram
if print_size_histogram:
print
print "Histograms of object sizes"
print
dumpbysize(bysizew, "written", "writes")
dumpbysize(bysize, "loaded", "loads")
def dumpbysize(bysize, how, how2):
print
print "Unique sizes %s: %s" % (how, addcommas(len(bysize)))
print "%10s %6s %6s" % ("size", "objs", how2)
sizes = bysize.keys()
sizes.sort()
for size in sizes:
loads = 0
for n in bysize[size].itervalues():
loads += n
print "%10s %6d %6d" % (addcommas(size),
len(bysize.get(size, "")),
loads)
def dumpbyinterval(byinterval, h0, he):
loads = 0
hits = 0
for code in byinterval.keys():
if code & 0x70 == 0x20:
n = byinterval[code]
loads += n
if code in (0x2A, 0x2C, 0x2E):
hits += n
if not loads:
return
if loads:
hr = 100.0 * hits / loads
else:
hr = 0.0
print "%s-%s %10s loads, %10s hits,%5.1f%% hit rate" % (
time.ctime(h0)[4:-8], time.ctime(he)[14:-8],
addcommas(loads), addcommas(hits), hr)
def hitrate(bycode):
loads = 0
hits = 0
for code in bycode.keys():
if code & 0x70 == 0x20:
n = bycode[code]
loads += n
if code in (0x2A, 0x2C, 0x2E):
hits += n
if loads:
return 100.0 * hits / loads
else:
return 0.0
def histogram(d):
bins = {}
for v in d.itervalues():
bins[v] = bins.get(v, 0) + 1
L = bins.items()
L.sort()
return L
def U64(s):
h, v = struct.unpack(">II", s)
return (long(h) << 32) + v
def addcommas(n):
sign, s = '', str(n)
if s[0] == '-':
sign, s = '-', s[1:]
i = len(s) - 3
while i > 0:
s = s[:i] + ',' + s[i:]
i -= 3
return sign + s
explain = {
# The first hex digit shows the operation, the second the outcome.
# If the second digit is in "02468" then it is a 'miss'.
# If it is in "ACE" then it is a 'hit'.
0x00: "_setup_trace (initialization)",
0x10: "invalidate (miss)",
0x1A: "invalidate (hit, version, writing 'n')",
0x1C: "invalidate (hit, writing 'i')",
0x20: "load (miss)",
0x22: "load (miss, version, status 'n')",
0x24: "load (miss, deleting index entry)",
0x26: "load (miss, no non-version data)",
0x28: "load (miss, version mismatch, no non-version data)",
0x2A: "load (hit, returning non-version data)",
0x2C: "load (hit, version mismatch, returning non-version data)",
0x2E: "load (hit, returning version data)",
0x3A: "update",
0x40: "modifiedInVersion (miss)",
0x4A: "modifiedInVersion (hit, return None, status 'n')",
0x4C: "modifiedInVersion (hit, return '')",
0x4E: "modifiedInVersion (hit, return version)",
0x5A: "store (non-version data present)",
0x5C: "store (only version data present)",
0x6A: "_copytocurrent",
0x70: "checkSize (cache flip)",
}
if __name__ == "__main__":
sys.exit(main())
=== Added File Zope/lib/python/ZEO/util.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Utilities for setting up the server environment."""
import os
def parentdir(p, n=1):
"""Return the parent of p, n levels up."""
d = p
while n:
d = os.path.split(d)[0]
if not d or d == '.':
d = os.getcwd()
n -= 1
return d
class Environment:
"""Determine location of the Data.fs & ZEO_SERVER.pid files.
Pass the argv[0] used to start ZEO to the constructor.
Use the zeo_pid and fs attributes to get the filenames.
"""
def __init__(self, argv0):
v = os.environ.get("INSTANCE_HOME")
if v is None:
# looking for a Zope/var directory assuming that this code
# is installed in Zope/lib/python/ZEO
p = parentdir(argv0, 4)
if os.path.isdir(os.path.join(p, "var")):
v = p
else:
v = os.getcwd()
self.home = v
self.var = os.path.join(v, "var")
if not os.path.isdir(self.var):
self.var = self.home
pid = os.environ.get("ZEO_SERVER_PID")
if pid is None:
pid = os.path.join(self.var, "ZEO_SERVER.pid")
self.zeo_pid = pid
self.fs = os.path.join(self.var, "Data.fs")
=== Added File Zope/lib/python/ZEO/version.txt ===
2.0b2