[Zope-Checkins] CVS: Zope/lib/python/ZEO - ClientCache.py:1.39.2.1 ClientStorage.py:1.76.2.1 ClientStub.py:1.10.4.1 CommitLog.py:1.4.8.1 Exceptions.py:1.5.8.1 ICache.py:1.3.8.1 README.txt:1.3.8.1 ServerStub.py:1.9.4.1 StorageServer.py:1.76.2.1 TransactionBuffer.py:1.8.4.1 __init__.py:1.13.2.1 simul.py:1.13.2.1 start.py:1.47.2.1 stats.py:1.18.2.1 util.py:1.4.2.1 version.txt:1.4.2.1

Chris McDonough chrism@zope.com
Tue, 8 Oct 2002 20:41:43 -0400


Update of /cvs-repository/Zope/lib/python/ZEO
In directory cvs.zope.org:/tmp/cvs-serv15249/ZEO

Added Files:
      Tag: chrism-install-branch
	ClientCache.py ClientStorage.py ClientStub.py CommitLog.py 
	Exceptions.py ICache.py README.txt ServerStub.py 
	StorageServer.py TransactionBuffer.py __init__.py simul.py 
	start.py stats.py util.py version.txt 
Log Message:
Committing ZEO to chrism-install-branch.


=== Added File Zope/lib/python/ZEO/ClientCache.py === (571/671 lines abridged)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################

# XXX TO DO
# use two indices rather than the sign bit of the index??????
# add a shared routine to read + verify a record???
# redesign header to include vdlen???
# rewrite the cache using a different algorithm???

"""Implement a client cache

The cache is managed as two files.

The cache can be persistent (meaning it is survives a process restart)
or temporary.  It is persistent if the client argument is not None.

Persistent cache files live in the var directory and are named
'c<storage>-<client>-<digit>.zec' where <storage> is the storage
argument (default '1'), <client> is the client argument, and <digit> is
0 or 1.  Temporary cache files are unnamed files in the standard
temporary directory as determined by the tempfile module.

The ClientStorage overrides the client name default to the value of
the environment variable ZEO_CLIENT, if it exists.

Each cache file has a 4-byte magic number followed by a sequence of
records of the form:

  offset in record: name -- description

  0: oid -- 8-byte object id

  8: status -- 1-byte status 'v': valid, 'n': non-version valid, 'i': invalid
               ('n' means only the non-version data in the record is valid)

  9: tlen -- 4-byte (unsigned) record length

  13: vlen -- 2-byte (unsigned) version length


[-=- -=- -=- 571 lines omitted -=- -=- -=-]

            if vlen+dlen+43+vdlen != tlen:
                rilog("inconsistent lengths", pos, fileindex)
                break
            seek(vdlen, 1)
            vs = read(8)
            if read(4) != h[9:13]:
                rilog("inconsistent tlen", pos, fileindex)
                break
        else:
            if h[8] in 'vn' and vlen == 0:
                if dlen+31 != tlen:
                    rilog("inconsistent nv lengths", pos, fileindex)
                seek(dlen, 1)
                if read(4) != h[9:13]:
                    rilog("inconsistent nv tlen", pos, fileindex)
                    break
            vs = None

        if h[8] in 'vn':
            if fileindex:
                index[oid] = -pos
            else:
                index[oid] = pos
            serial[oid] = h[-8:], vs
        else:
            if serial.has_key(oid):
                # We have a record for this oid, but it was invalidated!
                del serial[oid]
                del index[oid]


        pos = pos + tlen
        count += 1

    f.seek(pos)
    try:
        f.truncate()
    except:
        pass

    if count:
        log("read_index: cache file %d has %d records and %d bytes"
            % (fileindex, count, pos))

    return pos

def rilog(msg, pos, fileindex):
    # Helper to log messages from read_index
    log("read_index: %s at position %d in cache file %d"
        % (msg, pos, fileindex))


=== Added File Zope/lib/python/ZEO/ClientStorage.py === (694/794 lines abridged)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The ClientStorage class and the exceptions that it may raise.

Public contents of this module:

ClientStorage -- the main class, implementing the Storage API
ClientStorageError -- exception raised by ClientStorage
UnrecognizedResult -- exception raised by ClientStorage
ClientDisconnected -- exception raised by ClientStorage
"""

# XXX TO DO
# get rid of beginVerify, set up _tfile in verify_cache
# set self._storage = stub later, in endVerify
# if wait is given, wait until verify is complete

import cPickle
import os
import tempfile
import threading
import time

from ZEO import ClientCache, ServerStub
from ZEO.TransactionBuffer import TransactionBuffer
from ZEO.Exceptions import Disconnected
from ZEO.zrpc.client import ConnectionManager

from ZODB import POSException
from ZODB.TimeStamp import TimeStamp
from zLOG import LOG, PROBLEM, INFO, BLATHER

def log2(type, msg, subsys="ZCS:%d" % os.getpid()):
    LOG(subsys, type, msg)

try:
    from ZODB.ConflictResolution import ResolvedSerial
except ImportError:
    ResolvedSerial = 'rs'

[-=- -=- -=- 694 lines omitted -=- -=- -=-]

        # Queue an invalidate for the end the verification procedure.
        if self._pickler is None:
            # XXX This should never happen
            return
        self._pickler.dump(args)

    def endVerify(self):
        """Server callback to signal end of cache validation."""
        if self._pickler is None:
            return
        self._pickler.dump((0,0))
        self._tfile.seek(0)
        unpick = cPickle.Unpickler(self._tfile)
        f = self._tfile
        self._tfile = None

        while 1:
            oid, version = unpick.load()
            if not oid:
                break
            self._cache.invalidate(oid, version=version)
            self._db.invalidate(oid, version=version)
        f.close()

    def invalidateTrans(self, args):
        """Server callback to invalidate a list of (oid, version) pairs.

        This is called as the result of a transaction.
        """
        for oid, version in args:
            self._cache.invalidate(oid, version=version)
            try:
                self._db.invalidate(oid, version=version)
            except AttributeError, msg:
                log2(PROBLEM,
                    "Invalidate(%s, %s) failed for _db: %s" % (repr(oid),
                                                               repr(version),
                                                               msg))

    # Unfortunately, the ZEO 2 wire protocol uses different names for
    # several of the callback methods invoked by the StorageServer.
    # We can't change the wire protocol at this point because that
    # would require synchronized updates of clients and servers and we
    # don't want that.  So here we alias the old names to their new
    # implementations.

    begin = beginVerify
    invalidate = invalidateVerify
    end = endVerify
    Invalidate = invalidateTrans


=== Added File Zope/lib/python/ZEO/ClientStub.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""RPC stubs for interface exported by ClientStorage."""

class ClientStorage:

    """An RPC stub class for the interface exported by ClientStorage.

    This is the interface presented by ClientStorage to the
    StorageServer; i.e. the StorageServer calls these methods and they
    are executed in the ClientStorage.

    See the ClientStorage class for documentation on these methods.

    It is currently important that all methods here are asynchronous
    (meaning they don't have a return value and the caller doesn't
    wait for them to complete), *and* that none of them cause any
    calls from the client to the storage.  This is due to limitations
    in the zrpc subpackage.

    The on-the-wire names of some of the methods don't match the
    Python method names.  That's because the on-the-wire protocol was
    fixed for ZEO 2 and we don't want to change it.  There are some
    aliases in ClientStorage.py to make up for this.
    """

    def __init__(self, rpc):
        """Constructor.

        The argument is a connection: an instance of the
        zrpc.connection.Connection class.
        """
        self.rpc = rpc

    def beginVerify(self):
        self.rpc.callAsync('begin')

    def invalidateVerify(self, args):
        self.rpc.callAsync('invalidate', args)

    def endVerify(self):
        self.rpc.callAsync('end')

    def invalidateTrans(self, args):
        self.rpc.callAsync('Invalidate', args)

    def serialnos(self, arg):
        self.rpc.callAsync('serialnos', arg)

    def info(self, arg):
        self.rpc.callAsync('info', arg)


=== Added File Zope/lib/python/ZEO/CommitLog.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Log a transaction's commit info during two-phase commit.

A storage server allows multiple clients to commit transactions, but
must serialize them as the actually execute at the server.  The
concurrent commits are achieved by logging actions up until the
tpc_vote().  At that point, the entire transaction is committed on the
real storage.
"""
import cPickle
import tempfile

class CommitLog:

    def __init__(self):
        self.file = tempfile.TemporaryFile(suffix=".log")
        self.pickler = cPickle.Pickler(self.file, 1)
        self.pickler.fast = 1
        self.stores = 0
        self.read = 0

    def store(self, oid, serial, data, version):
        self.pickler.dump((oid, serial, data, version))
        self.stores += 1

    def get_loader(self):
        self.read = 1
        self.file.seek(0)
        return self.stores, cPickle.Unpickler(self.file)


=== Added File Zope/lib/python/ZEO/Exceptions.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Exceptions for ZEO."""

class Disconnected(Exception):
    """Exception raised when a ZEO client is disconnected from the
    ZEO server."""


=== Added File Zope/lib/python/ZEO/ICache.py ===
try:
    from Interface import Base
except ImportError:
    class Base:
        # a dummy interface for use when Zope's is unavailable
        pass

class ICache(Base):
    """ZEO client cache.

    __init__(storage, size, client, var)

    All arguments optional.

    storage -- name of storage
    size -- max size of cache in bytes
    client -- a string; if specified, cache is persistent.
    var -- var directory to store cache files in
    """

    def open():
        """Returns a sequence of object info tuples.

        An object info tuple is a pair containing an object id and a
        pair of serialnos, a non-version serialno and a version serialno:
        oid, (serial, ver_serial)

        This method builds an index of the cache and returns a
        sequence used for cache validation.
        """

    def close():
        """Closes the cache."""

    def verify(func):
        """Call func on every object in cache.

        func is called with three arguments
        func(oid, serial, ver_serial)
        """

    def invalidate(oid, version):
        """Remove object from cache."""

    def load(oid, version):
        """Load object from cache.

        Return None if object not in cache.
        Return data, serialno if object is in cache.
        """

    def store(oid, p, s, version, pv, sv):
        """Store a new object in the cache."""

    def update(oid, serial, version, data):
        """Update an object already in the cache.

        XXX This method is called to update objects that were modified by
        a transaction.  It's likely that it is already in the cache,
        and it may be possible for the implementation to operate more
        efficiently.
        """

    def modifiedInVersion(oid):
        """Return the version an object is modified in.

        '' signifies the trunk.
        Returns None if the object is not in the cache.
        """

    def checkSize(size):
        """Check if adding size bytes would exceed cache limit.

        This method is often called just before store or update.  The
        size is a hint about the amount of data that is about to be
        stored.  The cache may want to evict some data to make space.
        """


=== Added File Zope/lib/python/ZEO/README.txt ===
=======
ZEO 2.0
=======

What's ZEO?
===========

ZEO stands for Zope Enterprise Objects.  ZEO is an add-on for Zope
that allows a multiple processes to connect to a single ZODB storage.
Those processes can live on different machines, but don't need to.
ZEO 2 has many improvements over ZEO 1, and is incompatible with ZEO
1; if you upgrade an existing ZEO 1 installation, you must upgrade the
server and all clients simultaneous.  If you received ZEO 2 as part of
the ZODB 3 distribution, the ZEO 1 sources are provided in a separate
directory (ZEO1).  Some documentation for ZEO is available in the ZODB
3 package in the Doc subdirectory.  ZEO depends on the ZODB software;
it can be used with the version of ZODB distributed with Zope 2.5.1 or
later.  More information about ZEO can be found on its home on the
web:

    http://www.zope.org/Products/ZEO/

What's here?
============

This list of filenames is mostly for ZEO developers::

 ClientCache.py          client-side cache implementation
 ClientStorage.py        client-side storage implementation
 ClientStub.py           RPC stubs for callbacks from server to client
 CommitLog.py            buffer used during two-phase commit on the server
 Exceptions.py           definitions of exceptions
 ICache.py               interface definition for the client-side cache
 ServerStub.py           RPC stubs for the server
 StorageServer.py        server-side storage implementation
 TransactionBuffer.py    buffer used for transaction data in the client
 __init__.py             near-empty file to make this directory a package
 simul.py                command-line tool to simulate cache behavior
 start.py                command-line tool to start the storage server
 stats.py                command-line tool to process client cache traces
 tests/                  unit tests and other test utilities
 util.py                 utilities used by the server startup tool
 version.txt             text file indicating the ZEO version
 zrpc/                   subpackage implementing Remote Procedure Call (RPC)

Client Cache Tracing
====================

An important question for ZEO users is: how large should the ZEO
client cache be?  ZEO 2 (as of ZEO 2.0b2) has a new feature that lets
you collect a trace of cache activity and tools to analyze this trace,
enabling you to make an informed decision about the cache size.

Don't confuse the ZEO client cache with the Zope object cache.  The
ZEO client cache is only used when an object is not in the Zope object
cache; the ZEO client cache avoids roundtrips to the ZEO server.

Enabling Cache Tracing
----------------------

To enable cache tracing, set the environment variable ZEO_CACHE_TRACE
to the name of a file to which the ZEO client process can write.  If
the file doesn't exist, the ZEO will try to create it.  If there are
problems with the file, a log message is written to the standard Zope
log file.  To start or stop tracing, the ZEO client process (typically
a Zope application server) must be restarted.

The trace file can grow pretty quickly; on a moderately loaded server,
we observed it growing by 5 MB per hour.  The file consists of binary
records, each 24 bytes long; a detailed description of the record
lay-out is given in stats.py.  No sensitive data is logged.

Analyzing a Cache Trace
-----------------------

The stats.py command-line tool is the first-line tool to analyze a
cache trace.  Its default output consists of two parts: a one-line
summary of essential statistics for each segment of 15 minutes,
interspersed with lines indicating client restarts and "cache flip
events" (more about those later), followed by a more detailed summary
of overall statistics.

The most important statistic is probably the "hit rate", a percentage
indicating how many requests to load an object could be satisfied from
the cache.  Hit rates around 70% are good.  90% is probably close to
the theoretical maximum.  If you see a hit rate under 60% you can
probably improve the cache performance (and hence your Zope
application server's performance) by increasing the ZEO cache size.
This is normally configured using the cache_size keyword argument to
the ClientStorage() constructor in your custom_zodb.py file.  The
default cache size is 20 MB.

The stats.py tool shows its command line syntax when invoked without
arguments.  The tracefile argument can be a gzipped file if it has a
.gz extension.  It will read from stdin (assuming uncompressed data)
if the tracefile argument is '-'.

Simulating Different Cache Sizes
--------------------------------

Based on a cache trace file, you can make a prediction of how well the
cache might do with a different cache size.  The simul.py tool runs an
accurate simulation of the ZEO client cache implementation based upon
the events read from a trace file.  A new simulation is started each
time the trace file records a client restart event; if a trace file
contains more than one restart event, a separate line is printed for
each simulation, and line with overall statistics is added at the end.

Example, assuming the trace file is in /tmp/cachetrace.log::

    $ python simul.py -s 100 /tmp/cachetrace.log
      START TIME  DURATION    LOADS     HITS INVALS WRITES  FLIPS HITRATE
    Sep  4 11:59     38:01    59833    40473    257     20      2  67.6%
    $

This shows that with a 100 MB cache size, the cache hit rate is
67.6%.  So let's try this again with a 200 MB cache size::

    $ python simul.py -s 200 /tmp/cachetrace.log
      START TIME  DURATION    LOADS     HITS INVALS WRITES  FLIPS HITRATE
    Sep  4 11:59     38:01    59833    40921    258     20      1  68.4%
    $

This showed hardly any improvement.  So let's try a 300 MB cache
size::

    $ python2.0 simul.py -s 300 /tmp/cachetrace.log
    ZEOCacheSimulation, cache size 300,000,000 bytes
      START TIME  DURATION    LOADS     HITS INVALS WRITES  FLIPS HITRATE
    Sep  4 11:59     38:01    59833    40921    258     20      0  68.4%
    $ 

This shows that for this particular trace file, the maximum attainable
hit rate is 68.4%.  This is probably caused by the fact that nearly a
third of the objects mentioned in the trace were loaded only once --
the cache only helps if an object is loaded more than once.

The simul.py tool also supports simulating different cache
strategies.  Since none of these are implemented, these are not
further documented here.

Cache Flips
-----------

The cache uses two files, which are managed as follows:

  - Data are written to file 0 until file 0 exceeds limit/2 in size.

  - Data are written to file 1 until file 1 exceeds limit/2 in size.

  - File 0 is truncated to size 0 (or deleted and recreated).

  - Data are written to file 0 until file 0 exceeds limit/2 in size.

  - File 1 is truncated to size 0 (or deleted and recreated).

  - Data are written to file 1 until file 1 exceeds limit/2 in size.

and so on.

A switch from file 0 to file 1 is called a "cache flip".  At all cache
flips except the first, half of the cache contents is wiped out.  This
affects cache performance.  How badly this impact is can be seen from
the per-15-minutes summaries printed by stats.py.  The -i option lets
you choose a smaller summary interval which shows the impact more
acutely.

The simul.py tool shows the number of cache flips in the FLIPS column.
If you see more than one flip per hour the cache may be too small.


=== Added File Zope/lib/python/ZEO/ServerStub.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""RPC stubs for interface exported by StorageServer."""

class StorageServer:

    """An RPC stub class for the interface exported by ClientStorage.

    This is the interface presented by the StorageServer to the
    ClientStorage; i.e. the ClientStorage calls these methods and they
    are executed in the StorageServer.

    See the StorageServer module for documentation on these methods,
    with the exception of _update(), which is documented here.
    """

    def __init__(self, rpc):
        """Constructor.

        The argument is a connection: an instance of the
        zrpc.connection.Connection class.
        """
        self.rpc = rpc

    def _update(self):
        """Handle pending incoming messages.

        This method is typically only used when no asyncore mainloop
        is already active.  It can cause arbitrary callbacks from the
        server to the client to be handled.
        """
        self.rpc.pending()

    def register(self, storage_name, read_only):
        self.rpc.call('register', storage_name, read_only)

    def get_info(self):
        return self.rpc.call('get_info')

    def beginZeoVerify(self):
        self.rpc.callAsync('beginZeoVerify')

    def zeoVerify(self, oid, s, sv):
        self.rpc.callAsync('zeoVerify', oid, s, sv)

    def endZeoVerify(self):
        self.rpc.callAsync('endZeoVerify')

    def new_oids(self, n=None):
        if n is None:
            return self.rpc.call('new_oids')
        else:
            return self.rpc.call('new_oids', n)

    def pack(self, t, wait=None):
        if wait is None:
            self.rpc.call('pack', t)
        else:
            self.rpc.call('pack', t, wait)

    def zeoLoad(self, oid):
        return self.rpc.call('zeoLoad', oid)

    def storea(self, oid, serial, data, version, id):
        self.rpc.callAsync('storea', oid, serial, data, version, id)

    def tpc_begin(self, id, user, descr, ext, tid, status):
        return self.rpc.call('tpc_begin', id, user, descr, ext, tid, status)

    def vote(self, trans_id):
        return self.rpc.call('vote', trans_id)

    def tpc_finish(self, id):
        return self.rpc.call('tpc_finish', id)

    def tpc_abort(self, id):
        self.rpc.callAsync('tpc_abort', id)

    def abortVersion(self, src, id):
        return self.rpc.call('abortVersion', src, id)

    def commitVersion(self, src, dest, id):
        return self.rpc.call('commitVersion', src, dest, id)

    def history(self, oid, version, length=None):
        if length is None:
            return self.rpc.call('history', oid, version)
        else:
            return self.rpc.call('history', oid, version, length)

    def load(self, oid, version):
        return self.rpc.call('load', oid, version)

    def loadSerial(self, oid, serial):
        return self.rpc.call('loadSerial', oid, serial)

    def modifiedInVersion(self, oid):
        return self.rpc.call('modifiedInVersion', oid)

    def new_oid(self, last=None):
        if last is None:
            return self.rpc.call('new_oid')
        else:
            return self.rpc.call('new_oid', last)

    def store(self, oid, serial, data, version, trans):
        return self.rpc.call('store', oid, serial, data, version, trans)

    def transactionalUndo(self, trans_id, trans):
        return self.rpc.call('transactionalUndo', trans_id, trans)

    def undo(self, trans_id):
        return self.rpc.call('undo', trans_id)

    def undoLog(self, first, last):
        return self.rpc.call('undoLog', first, last)

    def undoInfo(self, first, last, spec):
        return self.rpc.call('undoInfo', first, last, spec)

    def versionEmpty(self, vers):
        return self.rpc.call('versionEmpty', vers)

    def versions(self, max=None):
        if max is None:
            return self.rpc.call('versions')
        else:
            return self.rpc.call('versions', max)


=== Added File Zope/lib/python/ZEO/StorageServer.py === (643/743 lines abridged)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The StorageServer class and the exception that it may raise.

This server acts as a front-end for one or more real storages, like
file storage or Berkeley storage.

XXX Need some basic access control-- a declaration of the methods
exported for invocation by the server.
"""

import asyncore
import cPickle
import os
import sys
import threading

from ZEO import ClientStub
from ZEO.CommitLog import CommitLog
from ZEO.zrpc.server import Dispatcher
from ZEO.zrpc.connection import ManagedServerConnection, Delay, MTDelay

import zLOG
from ZODB.POSException import StorageError, StorageTransactionError
from ZODB.POSException import TransactionError, ReadOnlyError
from ZODB.referencesf import referencesf
from ZODB.Transaction import Transaction

_label = "ZSS" # Default label used for logging.

def set_label():
    """Internal helper to reset the logging label (e.g. after fork())."""
    global _label
    _label = "ZSS:%s" % os.getpid()

def log(message, level=zLOG.INFO, label=None, error=None):
    """Internal helper to log a message using zLOG."""
    zLOG.LOG(label or _label, level, message, error=error)


[-=- -=- -=- 643 lines omitted -=- -=- -=-]

            oid, serial, data, version = loader.load()
            new_strategy.store(oid, serial, data, version)
        meth = getattr(new_strategy, self.name)
        return meth(*self.args)

    def abort(self, zeo_storage):
        # Delete (d, zeo_storage) from the _waiting list, if found.
        waiting = self.storage._waiting
        for i in range(len(waiting)):
            d, z = waiting[i]
            if z is zeo_storage:
                del waiting[i]
                break

def run_in_thread(method, *args):
    t = SlowMethodThread(method, args)
    t.start()
    return t.delay

class SlowMethodThread(threading.Thread):
    """Thread to run potentially slow storage methods.

    Clients can use the delay attribute to access the MTDelay object
    used to send a zrpc response at the right time.
    """

    # Some storage methods can take a long time to complete.  If we
    # run these methods via a standard asyncore read handler, they
    # will block all other server activity until they complete.  To
    # avoid blocking, we spawn a separate thread, return an MTDelay()
    # object, and have the thread reply() when it finishes.

    def __init__(self, method, args):
        threading.Thread.__init__(self)
        self._method = method
        self._args = args
        self.delay = MTDelay()

    def run(self):
        try:
            result = self._method(*self._args)
        except Exception:
            self.delay.error(sys.exc_info())
        else:
            self.delay.reply(result)

# Patch up class references
StorageServer.ZEOStorageClass = ZEOStorage
ZEOStorage.DelayedCommitStrategyClass = DelayedCommitStrategy
ZEOStorage.ImmediateCommitStrategyClass = ImmediateCommitStrategy


=== Added File Zope/lib/python/ZEO/TransactionBuffer.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A TransactionBuffer store transaction updates until commit or abort.

A transaction may generate enough data that it is not practical to
always hold pending updates in memory.  Instead, a TransactionBuffer
is used to store the data until a commit or abort.
"""

# A faster implementation might store trans data in memory until it
# reaches a certain size.

import cPickle
import tempfile
from threading import Lock

class TransactionBuffer:

    # Valid call sequences:
    #
    #     ((store | invalidate)* begin_iterate next* clear)* close
    #
    # get_size can be called any time

    # The TransactionBuffer is used by client storage to hold update
    # data until the tpc_finish().  It is normally used by a single
    # thread, because only one thread can be in the two-phase commit
    # at one time.

    # It is possible, however, for one thread to close the storage
    # while another thread is in the two-phase commit.  We must use
    # a lock to guard against this race, because unpredictable things
    # can happen in Python if one thread closes a file that another
    # thread is reading.  In a debug build, an assert() can fail.

    # XXX If an operation is performed on a closed TransactionBuffer,
    # it has no effect and does not raise an exception.  The only time
    # this should occur is when a ClientStorage is closed in one
    # thread while another thread is in its tpc_finish().  It's not
    # clear what should happen in this case.  If the tpc_finish()
    # completes without error, the Connection using it could have
    # inconsistent data.  This should have minimal effect, though,
    # because the Connection is connected to a closed storage.

    def __init__(self):
        self.file = tempfile.TemporaryFile(suffix=".tbuf")
        self.lock = Lock()
        self.closed = 0
        self.count = 0
        self.size = 0
        # It's safe to use a fast pickler because the only objects
        # stored are builtin types -- strings or None.
        self.pickler = cPickle.Pickler(self.file, 1)
        self.pickler.fast = 1

    def close(self):
        self.lock.acquire()
        try:
            self.closed = 1
            try:
                self.file.close()
            except OSError:
                pass
        finally:
            self.lock.release()

    def store(self, oid, version, data):
        self.lock.acquire()
        try:
            self._store(oid, version, data)
        finally:
            self.lock.release()

    def _store(self, oid, version, data):
        """Store oid, version, data for later retrieval"""
        if self.closed:
            return
        self.pickler.dump((oid, version, data))
        self.count += 1
        # Estimate per-record cache size
        self.size = self.size + len(data) + 31
        if version:
            # Assume version data has same size as non-version data
            self.size = self.size + len(version) + len(data) + 12

    def invalidate(self, oid, version):
        self.lock.acquire()
        try:
            if self.closed:
                return
            self.pickler.dump((oid, version, None))
            self.count += 1
        finally:
            self.lock.release()

    def clear(self):
        """Mark the buffer as empty"""
        self.lock.acquire()
        try:
            if self.closed:
                return
            self.file.seek(0)
            self.count = 0
            self.size = 0
        finally:
            self.lock.release()

    # unchecked constraints:
    # 1. can't call store() after begin_iterate()
    # 2. must call clear() after iteration finishes

    def begin_iterate(self):
        """Move the file pointer in advance of iteration"""
        self.lock.acquire()
        try:
            if self.closed:
                return
            self.file.flush()
            self.file.seek(0)
            self.unpickler = cPickle.Unpickler(self.file)
        finally:
            self.lock.release()

    def next(self):
        self.lock.acquire()
        try:
            return self._next()
        finally:
            self.lock.release()

    def _next(self):
        """Return next tuple of data or None if EOF"""
        if self.closed:
            return None
        if self.count == 0:
            del self.unpickler
            return None
        oid_ver_data = self.unpickler.load()
        self.count -= 1
        return oid_ver_data

    def get_size(self):
        """Return size of data stored in buffer (just a hint)."""

        return self.size


=== Added File Zope/lib/python/ZEO/__init__.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZEO -- Zope Enterprise Objects.

See the file README.txt in this directory for an overview.

ZEO's home on the web is

    http://www.zope.org/Products/ZEO/

"""

version = "2.0b2"


=== Added File Zope/lib/python/ZEO/simul.py === (638/738 lines abridged)
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Cache simulation.

Usage: simul.py [-bflyz] [-s size] tracefile

Use one of -b, -f, -l, -y or -z select the cache simulator:
-b: buddy system allocator
-f: simple free list allocator
-l: idealized LRU (no allocator)
-y: variation on the existing ZEO cache that copies to current file
-z: existing ZEO cache (default)

Options:
-s size: cache size in MB (default 20 MB)

Note: the buddy system allocator rounds the cache size up to a power of 2
"""

import sys
import time
import getopt
import struct

def usage(msg):
    print >>sys.stderr, msg
    print >>sys.stderr, __doc__

def main():
    # Parse options
    MB = 1000*1000
    cachelimit = 20*MB
    simclass = ZEOCacheSimulation
    try:
        opts, args = getopt.getopt(sys.argv[1:], "bflyzs:")
    except getopt.error, msg:
        usage(msg)
        return 2

[-=- -=- -=- 638 lines omitted -=- -=- -=-]

    queue = []
    T = 0
    blocks = 0
    while T < 5000:
        while queue and queue[0][0] <= T:
            time, node = heapq.heappop(queue)
            assert time == T
            ##print "free addr=%d, size=%d" % (node.addr, node.size)
            cache.free(node)
            blocks -= 1
        size = random.randint(100, 2000)
        lifetime = random.randint(1, 100)
        node = cache.alloc(size)
        if node is None:
            print "out of mem"
            cache.dump("T=%4d: %d blocks;" % (T, blocks))
            break
        else:
            ##print "alloc addr=%d, size=%d" % (node.addr, node.size)
            blocks += 1
            heapq.heappush(queue, (T + lifetime, node))
        T = T+1
        if T % reportfreq == 0:
            cache.dump("T=%4d: %d blocks;" % (T, blocks))

def hitrate(loads, hits):
    return "%5.1f%%" % (100.0 * hits / max(1, loads))

def duration(secs):

    mm, ss = divmod(secs, 60)
    hh, mm = divmod(mm, 60)
    if hh:
        return "%d:%02d:%02d" % (hh, mm, ss)
    if mm:
        return "%d:%02d" % (mm, ss)
    return "%d" % ss

def addcommas(n):
    sign, s = '', str(n)
    if s[0] == '-':
        sign, s = '-', s[1:]
    i = len(s) - 3
    while i > 0:
        s = s[:i] + ',' + s[i:]
        i -= 3
    return sign + s

if __name__ == "__main__":
    sys.exit(main())


=== Added File Zope/lib/python/ZEO/start.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Start the ZEO storage server."""

from __future__ import nested_scopes

import sys, os, getopt
import types

def directory(p, n=1):
    d = p
    while n:
        d = os.path.split(d)[0]
        if not d or d == '.':
            d = os.getcwd()
        n -= 1
    return d

def get_storage(m, n, cache={}):
    p = sys.path
    d, m = os.path.split(m)
    if m.endswith('.py'):
        m = m[:-3]
    im = cache.get((d, m))
    if im is None:
        if d:
            p = [d] + p
        import imp
        im = imp.find_module(m, p)
        im = imp.load_module(m, *im)
        cache[(d, m)] = im
    return getattr(im, n)

def set_uid(arg):
    """Try to set uid and gid based on -u argument.

    This will only work if this script is run by root.
    """
    try:
        import pwd
    except ImportError:
        LOG('ZEO/start.py', INFO, ("Can't set uid to %s."
                                "pwd module is not available." % arg))
        return
    try:
        gid = None
        try:
            arg = int(arg)
        except: # conversion could raise all sorts of errors
            uid = pwd.getpwnam(arg)[2]
            gid = pwd.getpwnam(arg)[3]
        else:
            uid = pwd.getpwuid(arg)[2]
            gid = pwd.getpwuid(arg)[3]
        if gid is not None:
            try:
                os.setgid(gid)
            except OSError:
                pass
        try:
            os.setuid(uid)
        except OSError:
            pass
    except KeyError:
        LOG('ZEO/start.py', ERROR, ("can't find uid %s" % arg))

def setup_signals(storages):
    try:
        import signal
    except ImportError:
        return

    try:
        xfsz = signal.SIFXFSZ
    except AttributeError:
        pass
    else:
        signal.signal(xfsz, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, lambda sig, frame: shutdown(storages))
    signal.signal(signal.SIGINT, lambda sig, frame: shutdown(storages, 0))
    try:
        signal.signal(signal.SIGHUP, rotate_logs_handler)
    except:
        pass

def main(argv):
    me = argv[0]
    sys.path.insert(0, directory(me, 2))

    global LOG, INFO, ERROR
    from zLOG import LOG, INFO, ERROR, PANIC
    from ZEO.util import Environment
    env = Environment(me)

    # XXX hack for profiling support
    global unix, storages, asyncore

    args = []
    last = ''
    for a in argv[1:]:
        if (a[:1] != '-' and a.find('=') > 0 and last != '-S'): # lame, sorry
            a = a.split("=")
            os.environ[a[0]] = "=".join(a[1:])
            continue
        args.append(a)
        last = a

    usage="""%s [options] [filename]

    where options are:

       -D -- Run in debug mode

       -d -- Set STUPD_LOG_SEVERITY to -300

       -U -- Unix-domain socket file to listen on

       -u username or uid number

         The username to run the ZEO server as. You may want to run
         the ZEO server as 'nobody' or some other user with limited
         resouces. The only works under Unix, and if ZServer is
         started by root.

       -p port -- port to listen on

       -h adddress -- host address to listen on

       -s -- Don't use zdeamon

       -S storage_name=module_path:attr_name -- A storage specification

          where:

            storage_name -- is the storage name used in the ZEO protocol.
               This is the name that you give as the optional
               'storage' keyword argument to the ClientStorage constructor.

            module_path -- This is the path to a Python module
               that defines the storage object(s) to be served.
               The module path should omit the prefix (e.g. '.py').

            attr_name -- This is the name to which the storage object
              is assigned in the module.

       -P file -- Run under profile and dump output to file.  Implies the
          -s flag.

    if no file name is specified, then %s is used.
    """ % (me, env.fs)

    try:
        opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:P:d')
    except getopt.error, msg:
        print usage
        print msg
        sys.exit(1)

    port = None
    debug = 0
    host = ''
    unix = None
    Z = 1
    UID = 'nobody'
    prof = None
    detailed = 0
    fs = None
    for o, v in opts:
        if o =='-p':
            port = int(v)
        elif o =='-h':
            host = v
        elif o =='-U':
            unix = v
        elif o =='-u':
            UID = v
        elif o =='-D':
            debug = 1
        elif o =='-d':
            detailed = 1
        elif o =='-s':
            Z = 0
        elif o =='-P':
            prof = v

    if prof:
        Z = 0

    if port is None and unix is None:
        print usage
        print 'No port specified.'
        sys.exit(1)

    if args:
        if len(args) > 1:
            print usage
            print 'Unrecognizd arguments: ', " ".join(args[1:])
            sys.exit(1)
        fs = args[0]

    if debug:
        os.environ['Z_DEBUG_MODE'] = '1'
    if detailed:
        os.environ['STUPID_LOG_SEVERITY'] = '-300'

    set_uid(UID)

    if Z:
        try:
            import posix
        except:
            pass
        else:
            import zdaemon
            zdaemon.run(sys.argv, '')

    try:

        import ZEO.StorageServer, asyncore

        storages = {}
        for o, v in opts:
            if o == '-S':
                n, m = v.split("=", 1)
                if m.find(":") >= 0:
                    # we got an attribute name
                    m, a = m.split(':')
                else:
                    # attribute name must be same as storage name
                    a=n
                storages[n]=get_storage(m,a)

        if not storages:
            from ZODB.FileStorage import FileStorage
            storages['1'] = FileStorage(fs or env.fs)

        # Try to set up a signal handler
        setup_signals(storages)

        items = storages.items()
        items.sort()
        for kv in items:
            LOG('ZEO/start.py', INFO, 'Serving %s:\t%s' % kv)

        if not unix:
            unix = host, port

        ZEO.StorageServer.StorageServer(unix, storages)

        try:
            ppid, pid = os.getppid(), os.getpid()
        except:
            pass # getpid not supported
        else:
            f = open(env.zeo_pid, 'w')
            f.write("%s %s\n" % (ppid, pid))
            f.close()

    except:
        # Log startup exception and tell zdaemon not to restart us.
        info = sys.exc_info()
        try:
            LOG("ZEO/start.py", PANIC, "Startup exception", error=info)
        except:
            pass

        import traceback
        traceback.print_exception(*info)

        sys.exit(0)

    try:
        try:
            asyncore.loop()
        finally:
            if os.path.isfile(env.zeo_pid):
                os.unlink(env.zeo_pid)
    except SystemExit:
        raise
    except:
        info = sys.exc_info()
        try:
            LOG("ZEO/start.py", PANIC, "Unexpected error", error=info)
        except:
            pass
        import traceback
        traceback.print_exception(*info)
        sys.exit(1)

def rotate_logs():
    import zLOG
    # There hasn't been a clear way to reinitialize the MinimalLogger.
    # I'll checkin the public initialize() method soon, but also try some
    # other strategies for older Zope installs :-(.
    init = getattr(zLOG, 'initialize', None)
    if init is not None:
        init()
        return
    # This will work if the minimal logger is in use, but not if some
    # other logger is active.
    import zLOG.MinimalLogger
    zLOG.MinimalLogger._log.initialize()

def rotate_logs_handler(signum, frame):
    rotate_logs()

    import signal
    signal.signal(signal.SIGHUP, rotate_logs_handler)

def shutdown(storages, die=1):
    LOG("ZEO/start.py", INFO, "Received signal")
    import asyncore

    # Do this twice, in case we got some more connections
    # while going through the loop.  This is really sort of
    # unnecessary, since we now use so_reuseaddr.
    for ignored in 1,2:
        for socket in asyncore.socket_map.values():
            try:
                socket.close()
            except:
                pass

    for storage in storages.values():
        try:
            storage.close()
        finally:
            pass

    try:
        s = die and "shutdown" or "restart"
        LOG('ZEO/start.py', INFO, "Shutting down (%s)" % s)
    except:
        pass

    if die:
        sys.exit(0)
    else:
        sys.exit(1)

if __name__=='__main__':
    main(sys.argv)


=== Added File Zope/lib/python/ZEO/stats.py ===
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Trace file statistics analyzer.

Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] tracefile
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
"""

"""File format:

Each record is 24 bytes, with the following layout.  Numbers are
big-endian integers.

Offset  Size  Contents

0       4     timestamp (seconds since 1/1/1970)
4       3     data size, in 256-byte increments, rounded up
7       1     code (see below)
8       8     object id
16      8     serial number

The code at offset 7 packs three fields:

Mask    bits  Contents

0x80    1     set if there was a non-empty version string
0x7e    6     function and outcome code
0x01    1     current cache file (0 or 1)

The function and outcome codes are documented in detail at the end of
this file in the 'explain' dictionary.  Note that the keys there (and
also the arguments to _trace() in ClientStorage.py) are 'code & 0x7e',
i.e. the low bit is always zero.
"""

import sys
import time
import getopt
import struct

def usage(msg):
    print >>sys.stderr, msg
    print >>sys.stderr, __doc__

def main():
    # Parse options
    verbose = 0
    quiet = 0
    dostats = 1
    print_size_histogram = 0
    print_histogram = 0
    interval = 900 # Every 15 minutes
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hi:qsSv")
    except getopt.error, msg:
        usage(msg)
        return 2
    for o, a in opts:
        if o == '-h':
            print_histogram = 1
        if o == "-i":
            interval = int(60 * float(a))
            if interval <= 0:
                interval = 60
            elif interval > 3600:
                interval = 3600
        if o == "-q":
            quiet = 1
            verbose = 0
        if o == "-s":
            print_size_histogram = 1
        if o == "-S":
            dostats = 0
        if o == "-v":
            verbose = 1
    if len(args) != 1:
        usage("exactly one file argument required")
        return 2
    filename = args[0]

    # Open file
    if filename.endswith(".gz"):
        # Open gzipped file
        try:
            import gzip
        except ImportError:
            print >>sys.stderr,  "can't read gzipped files (no module gzip)"
            return 1
        try:
            f = gzip.open(filename, "rb")
        except IOError, msg:
            print >>sys.stderr,  "can't open %s: %s" % (filename, msg)
            return 1
    elif filename == '-':
        # Read from stdin
        f = sys.stdin
    else:
        # Open regular file
        try:
            f = open(filename, "rb")
        except IOError, msg:
            print >>sys.stderr,  "can't open %s: %s" % (filename, msg)
            return 1

    # Read file, gathering statistics, and printing each record if verbose
    rt0 = time.time()
    bycode = {}
    records = 0
    versions = 0
    t0 = te = None
    datarecords = 0
    datasize = 0L
    file0 = file1 = 0
    oids = {}
    bysize = {}
    bysizew = {}
    total_loads = 0
    byinterval = {}
    thisinterval = None
    h0 = he = None
    offset = 0
    f_read = f.read
    struct_unpack = struct.unpack
    try:
        while 1:
            r = f_read(8)
            if len(r) < 8:
                break
            offset += 8
            ts, code = struct_unpack(">ii", r)
            if ts == 0:
                # Must be a misaligned record caused by a crash
                if not quiet:
                    print "Skipping 8 bytes at offset", offset-8
                continue
            r = f_read(16)
            if len(r) < 16:
                break
            offset += 16
            records += 1
            oid, serial = struct_unpack(">8s8s", r)
            if t0 is None:
                t0 = ts
                thisinterval = t0 / interval
                h0 = he = ts
            te = ts
            if ts / interval != thisinterval:
                if not quiet:
                    dumpbyinterval(byinterval, h0, he)
                byinterval = {}
                thisinterval = ts / interval
                h0 = ts
            he = ts
            dlen, code = code & 0x7fffff00, code & 0xff
            if dlen:
                datarecords += 1
                datasize += dlen
            version = '-'
            if code & 0x80:
                version = 'V'
                versions += 1
            current = code & 1
            if current:
                file1 += 1
            else:
                file0 += 1
            code = code & 0x7e
            bycode[code] = bycode.get(code, 0) + 1
            byinterval[code] = byinterval.get(code, 0) + 1
            if dlen:
                if code & 0x70 == 0x20: # All loads
                    bysize[dlen] = d = bysize.get(dlen) or {}
                    d[oid] = d.get(oid, 0) + 1
                elif code == 0x3A: # Update
                    bysizew[dlen] = d = bysizew.get(dlen) or {}
                    d[oid] = d.get(oid, 0) + 1
            if verbose:
                print "%s %d %02x %016x %016x %1s %s" % (
                    time.ctime(ts)[4:-5],
                    current,
                    code,
                    U64(oid),
                    U64(serial),
                    version,
                    dlen and str(dlen) or "")
            if code & 0x70 == 0x20:
                oids[oid] = oids.get(oid, 0) + 1
                total_loads += 1
            if code in (0x00, 0x70):
                if not quiet:
                    dumpbyinterval(byinterval, h0, he)
                byinterval = {}
                thisinterval = ts / interval
                h0 = he = ts
                if not quiet:
                    print time.ctime(ts)[4:-5],
                    if code == 0x00:
                        print '='*20, "Restart", '='*20
                    else:
                        print '-'*20, "Flip->%d" % current, '-'*20
    except KeyboardInterrupt:
        print "\nInterrupted.  Stats so far:\n"

    f.close()
    rte = time.time()
    if not quiet:
        dumpbyinterval(byinterval, h0, he)

    # Error if nothing was read
    if not records:
        print >>sys.stderr, "No records processed"
        return 1

    # Print statistics
    if dostats:
        print
        print "Read %s records (%s bytes) in %.1f seconds" % (
            addcommas(records), addcommas(records*24), rte-rt0)
        print "Versions:   %s records used a version" % addcommas(versions)
        print "First time: %s" % time.ctime(t0)
        print "Last time:  %s" % time.ctime(te)
        print "Duration:   %s seconds" % addcommas(te-t0)
        print "File stats: %s in file 0; %s in file 1" % (
            addcommas(file0), addcommas(file1))
        print "Data recs:  %s (%.1f%%), average size %.1f KB" % (
            addcommas(datarecords),
            100.0 * datarecords / records,
            datasize / 1024.0 / datarecords)
        print "Hit rate:   %.1f%% (load hits / loads)" % hitrate(bycode)
        print
        codes = bycode.keys()
        codes.sort()
        print "%13s %4s %s" % ("Count", "Code", "Function (action)")
        for code in codes:
            print "%13s  %02x  %s" % (
                addcommas(bycode.get(code, 0)),
                code,
                explain.get(code) or "*** unknown code ***")

    # Print histogram
    if print_histogram:
        print
        print "Histogram of object load frequency"
        total = len(oids)
        print "Unique oids: %s" % addcommas(total)
        print "Total loads: %s" % addcommas(total_loads)
        s = addcommas(total)
        width = max(len(s), len("objects"))
        fmt = "%5d %" + str(width) + "s %5.1f%% %5.1f%% %5.1f%%"
        hdr = "%5s %" + str(width) + "s %6s %6s %6s"
        print hdr % ("loads", "objects", "%obj", "%load", "%cum")
        cum = 0.0
        for binsize, count in histogram(oids):
            obj_percent = 100.0 * count / total
            load_percent = 100.0 * count * binsize / total_loads
            cum += load_percent
            print fmt % (binsize, addcommas(count),
                         obj_percent, load_percent, cum)

    # Print size histogram
    if print_size_histogram:
        print
        print "Histograms of object sizes"
        print
        dumpbysize(bysizew, "written", "writes")
        dumpbysize(bysize, "loaded", "loads")

def dumpbysize(bysize, how, how2):
    print
    print "Unique sizes %s: %s" % (how, addcommas(len(bysize)))
    print "%10s %6s %6s" % ("size", "objs", how2)
    sizes = bysize.keys()
    sizes.sort()
    for size in sizes:
        loads = 0
        for n in bysize[size].itervalues():
            loads += n
        print "%10s %6d %6d" % (addcommas(size),
                                len(bysize.get(size, "")),
                                loads)

def dumpbyinterval(byinterval, h0, he):
    loads = 0
    hits = 0
    for code in byinterval.keys():
        if code & 0x70 == 0x20:
            n = byinterval[code]
            loads += n
            if code in (0x2A, 0x2C, 0x2E):
                hits += n
    if not loads:
        return
    if loads:
        hr = 100.0 * hits / loads
    else:
        hr = 0.0
    print "%s-%s %10s loads, %10s hits,%5.1f%% hit rate" % (
        time.ctime(h0)[4:-8], time.ctime(he)[14:-8],
        addcommas(loads), addcommas(hits), hr)

def hitrate(bycode):
    loads = 0
    hits = 0
    for code in bycode.keys():
        if code & 0x70 == 0x20:
            n = bycode[code]
            loads += n
            if code in (0x2A, 0x2C, 0x2E):
                hits += n
    if loads:
        return 100.0 * hits / loads
    else:
        return 0.0

def histogram(d):
    bins = {}
    for v in d.itervalues():
        bins[v] = bins.get(v, 0) + 1
    L = bins.items()
    L.sort()
    return L

def U64(s):
    h, v = struct.unpack(">II", s)
    return (long(h) << 32) + v

def addcommas(n):
    sign, s = '', str(n)
    if s[0] == '-':
        sign, s = '-', s[1:]
    i = len(s) - 3
    while i > 0:
        s = s[:i] + ',' + s[i:]
        i -= 3
    return sign + s

explain = {
    # The first hex digit shows the operation, the second the outcome.
    # If the second digit is in "02468" then it is a 'miss'.
    # If it is in "ACE" then it is a 'hit'.

    0x00: "_setup_trace (initialization)",

    0x10: "invalidate (miss)",
    0x1A: "invalidate (hit, version, writing 'n')",
    0x1C: "invalidate (hit, writing 'i')",

    0x20: "load (miss)",
    0x22: "load (miss, version, status 'n')",
    0x24: "load (miss, deleting index entry)",
    0x26: "load (miss, no non-version data)",
    0x28: "load (miss, version mismatch, no non-version data)",
    0x2A: "load (hit, returning non-version data)",
    0x2C: "load (hit, version mismatch, returning non-version data)",
    0x2E: "load (hit, returning version data)",

    0x3A: "update",

    0x40: "modifiedInVersion (miss)",
    0x4A: "modifiedInVersion (hit, return None, status 'n')",
    0x4C: "modifiedInVersion (hit, return '')",
    0x4E: "modifiedInVersion (hit, return version)",

    0x5A: "store (non-version data present)",
    0x5C: "store (only version data present)",

    0x6A: "_copytocurrent",

    0x70: "checkSize (cache flip)",
    }

if __name__ == "__main__":
    sys.exit(main())


=== Added File Zope/lib/python/ZEO/util.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Utilities for setting up the server environment."""

import os

def parentdir(p, n=1):
    """Return the parent of p, n levels up."""
    d = p
    while n:
        d = os.path.split(d)[0]
        if not d or d == '.':
            d = os.getcwd()
        n -= 1
    return d

class Environment:
    """Determine location of the Data.fs & ZEO_SERVER.pid files.

    Pass the argv[0] used to start ZEO to the constructor.

    Use the zeo_pid and fs attributes to get the filenames.
    """

    def __init__(self, argv0):
        v = os.environ.get("INSTANCE_HOME")
        if v is None:
            # looking for a Zope/var directory assuming that this code
            # is installed in Zope/lib/python/ZEO
            p = parentdir(argv0, 4)
            if os.path.isdir(os.path.join(p, "var")):
                v = p
            else:
                v = os.getcwd()
        self.home = v
        self.var = os.path.join(v, "var")
        if not os.path.isdir(self.var):
            self.var = self.home

        pid = os.environ.get("ZEO_SERVER_PID")
        if pid is None:
            pid = os.path.join(self.var, "ZEO_SERVER.pid")

        self.zeo_pid = pid
        self.fs = os.path.join(self.var, "Data.fs")


=== Added File Zope/lib/python/ZEO/version.txt ===
2.0b2