[Zodb-checkins] CVS: Zope/lib/python/ZEO - runsvr.py:1.14.2.1 ClientCache.py:1.39.2.2 ClientStorage.py:1.76.2.2 README.txt:1.3.8.2 ServerStub.py:1.9.4.2 StorageServer.py:1.76.2.2 __init__.py:1.13.2.2 simul.py:1.13.2.3 start.py:1.47.2.3 stats.py:1.18.2.2 version.txt:1.4.2.2
Chris McDonough
chrism@zope.com
Sun, 24 Nov 2002 18:55:57 -0500
Update of /cvs-repository/Zope/lib/python/ZEO
In directory cvs.zope.org:/tmp/cvs-serv13982
Modified Files:
Tag: chrism-install-branch
ClientCache.py ClientStorage.py README.txt ServerStub.py
StorageServer.py __init__.py simul.py start.py stats.py
version.txt
Added Files:
Tag: chrism-install-branch
runsvr.py
Log Message:
Merge with HEAD.
=== Added File Zope/lib/python/ZEO/runsvr.py ===
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Start the ZEO storage server.
Usage: %s [-C URL] [-a ADDRESS] [-f FILENAME] [-h]
Options:
-C/--configuration URL -- configuration file or URL
-a/--address ADDRESS -- server address of the form PORT, HOST:PORT, or PATH
(a PATH must contain at least one "/")
-f/--filename FILENAME -- filename for FileStorage
-h/--help -- print this usage message and exit
Unless -C is specified, -a and -f are required.
"""
# The code here is designed to be reused by other, similar servers.
# For the forseeable future, it must work under Python 2.1 as well as
# 2.2 and above.
# XXX The option parsing infrastructure could be shared with zdaemon.py
import os
import sys
import getopt
import signal
import socket
import zLOG
import ZConfig
import ZConfig.Common
import ZODB.StorageConfig
class Options:
"""A class to parse and hold the command line options.
Options are represented by various attributes (zeoport etc.).
Positional arguments are represented by the args attribute.
This also has a public usage() method that can be used to report
errors related to the command line.
"""
configuration = None
rootconf = None
args = []
def __init__(self, args=None, progname=None, doc=None):
"""Constructor.
Optional arguments:
args -- the command line arguments, less the program name
(default is sys.argv[1:] at the time of call)
progname -- the program name (default sys.argv[0])
doc -- usage message (default, __main__.__doc__)
"""
if args is None:
args = sys.argv[1:]
if progname is None:
progname = sys.argv[0]
self.progname = progname
if doc is None:
import __main__
doc = __main__.__doc__
if doc and not doc.endswith("\n"):
doc += "\n"
self.doc = doc
try:
self.options, self.args = getopt.getopt(args,
self._short_options,
self._long_options)
except getopt.error, msg:
self.usage(str(msg))
for opt, arg in self.options:
self.handle_option(opt, arg)
self.check_options()
# Default set of options. Subclasses should override.
_short_options = "C:h"
_long_options = ["--configuration=", "--help"]
def handle_option(self, opt, arg):
"""Handle one option. Subclasses should override.
This sets the various instance variables overriding the defaults.
When -h is detected, print the module docstring to stdout and exit(0).
"""
if opt in ("-C", "--configuration"):
self.set_configuration(arg)
if opt in ("-h", "--help"):
self.help()
def set_configuration(self, arg):
self.configuration = arg
def check_options(self):
"""Check options. Subclasses may override.
This can be used to ensure certain options are set, etc.
"""
self.load_configuration()
def load_configuration(self):
if self.rootconf or not self.configuration:
return
self.rootconf = ZConfig.load(self.configuration)
def help(self):
"""Print a long help message (self.doc) to stdout and exit(0).
Occurrences of "%s" in self.doc are replaced by self.progname.
"""
doc = self.doc
if doc.find("%s") > 0:
doc = doc.replace("%s", self.progname)
print doc
sys.exit(0)
def usage(self, msg):
"""Print a brief error message to stderr and exit(2)."""
sys.stderr.write("Error: %s\n" % str(msg))
sys.stderr.write("For help, use %s -h\n" % self.progname)
sys.exit(2)
class ZEOOptions(Options):
hostname = None # A subclass may set this
hostconf = None # <Host> section
zeoconf = None # <ZEO> section
logconf = None # <Log> section
family = None # set by -a; AF_UNIX or AF_INET
address = None # set by -a; string or (host, port)
storages = None # set by -f
_short_options = "a:C:f:h"
_long_options = [
"--address=",
"--configuration=",
"--filename=",
"--help",
]
def handle_option(self, opt, arg):
# Alphabetical order please!
if opt in ("-a", "--address"):
if "/" in arg:
self.family = socket.AF_UNIX
self.address = arg
else:
self.family = socket.AF_INET
if ":" in arg:
host, port = arg.split(":", 1)
else:
host = ""
port = arg
try:
port = int(port)
except: # int() can raise all sorts of errors
self.usage("invalid port number: %r" % port)
self.address = (host, port)
elif opt in ("-f", "--filename"):
from ZODB.FileStorage import FileStorage
if not self.storages:
self.storages = {}
key = str(1 + len(self.storages))
self.storages[key] = (FileStorage, {"file_name": arg})
else:
# Pass it to the base class, for --help/-h
Options.handle_option(self, opt, arg)
def check_options(self):
Options.check_options(self) # Calls load_configuration()
if not self.storages:
self.usage("no storages specified; use -f or -C")
if self.family is None:
self.usage("no server address specified; use -a or -C")
if self.args:
self.usage("positional arguments are not supported")
def load_configuration(self):
Options.load_configuration(self) # Sets self.rootconf
if not self.rootconf:
return
try:
self.hostconf = self.rootconf.getSection("Host")
except ZConfig.Common.ConfigurationConflictingSectionError:
if not self.hostname:
self.hostname = socket.getfqdn()
self.hostconf = self.rootconf.getSection("Host", self.hostname)
if self.hostconf is None:
# If no <Host> section exists, fall back to the root
self.hostconf = self.rootconf
self.zeoconf = self.hostconf.getSection("ZEO")
if self.zeoconf is None:
# If no <ZEO> section exists, fall back to the host (or root)
self.zeoconf = self.hostconf
self.logconf = self.hostconf.getSection("Log")
# Now extract options from various configuration sections
self.load_zeoconf()
self.load_logconf()
self.load_storages()
def load_zeoconf(self):
# Get some option defaults from the configuration
if self.family:
# -a option overrides
return
port = self.zeoconf.getint("server-port")
path = self.zeoconf.get("path")
if port and path:
self.usage(
"Configuration contains conflicting ZEO information:\n"
"Exactly one of 'path' and 'server-port' may be given.")
if port:
host = self.hostconf.get("hostname", "")
self.family = socket.AF_INET
self.address = (host, port)
elif path:
self.family = socket.AF_UNIX
self.address = path
def load_logconf(self):
# Get logging options from conf, unless overridden by environment
if not self.logconf:
return
reinit = 0
if os.getenv("EVENT_LOG_FILE") is None:
if os.getenv("STUPID_LOG_FILE") is None:
path = self.logconf.get("path")
if path is not None:
os.environ["EVENT_LOG_FILE"] = path
os.environ["STUPID_LOG_FILE"] = path
reinit = 1
if os.getenv("EVENT_LOG_SEVERITY") is None:
if os.getenv("STUPID_LOG_SEVERITY") is None:
level = self.logconf.get("level")
if level is not None:
os.environ["EVENT_LOG_SEVERITY"] = level
os.environ["STUPID_LOG_SEVERITY"] = level
reinit = 1
if reinit:
zLOG.initialize()
def load_storages(self):
# Get the storage specifications
if self.storages:
# -f option overrides
return
storagesections = self.zeoconf.getChildSections("Storage")
self.storages = {}
for section in storagesections:
name = section.name
if not name:
name = str(1 + len(self.storages))
if self.storages.has_key(name):
# (Actually, the parser doesn't allow this)
self.usage("duplicate storage name %r" % name)
self.storages[name] = ZODB.StorageConfig.getStorageInfo(section)
class ZEOServer:
OptionsClass = ZEOOptions
def __init__(self, options=None):
if options is None:
options = self.OptionsClass()
self.options = options
def main(self):
self.check_socket()
self.clear_socket()
try:
self.open_storages()
self.setup_signals()
self.create_server()
self.loop_forever()
finally:
self.close_storages()
self.clear_socket()
def check_socket(self):
if self.can_connect(self.options.family, self.options.address):
self.options.usage("address %s already in use" %
repr(self.options.address))
def can_connect(self, family, address):
s = socket.socket(family, socket.SOCK_STREAM)
try:
s.connect(address)
except socket.error:
return 0
else:
s.close()
return 1
def clear_socket(self):
if isinstance(self.options.address, type("")):
try:
os.unlink(self.options.address)
except os.error:
pass
def open_storages(self):
self.storages = {}
for name, (cls, args) in self.options.storages.items():
info("open storage %r: %s.%s(**%r)" %
(name, cls.__module__, cls.__name__, args))
self.storages[name] = cls(**args)
def setup_signals(self):
"""Set up signal handlers.
The signal handler for SIGFOO is a method handle_sigfoo().
If no handler method is defined for a signal, the signal
action is not changed from its initial value. The handler
method is called without additional arguments.
"""
if os.name != "posix":
return
if hasattr(signal, 'SIGXFSZ'):
signal.signal(signal.SIGXFSZ, signal.SIG_IGN) # Special case
init_signames()
for sig, name in signames.items():
method = getattr(self, "handle_" + name.lower(), None)
if method is not None:
def wrapper(sig_dummy, frame_dummy, method=method):
method()
signal.signal(sig, wrapper)
def create_server(self):
from ZEO.StorageServer import StorageServer
self.server = StorageServer(self.options.address, self.storages)
def loop_forever(self):
import asyncore
asyncore.loop()
def handle_sigterm(self):
info("terminated by SIGTERM")
sys.exit(0)
def handle_sigint(self):
info("terminated by SIGINT")
sys.exit(0)
def handle_sigusr2(self):
# This requires a modern zLOG (from Zope 2.6 or later); older
# zLOG packages don't have the initialize() method
info("reinitializing zLOG")
# XXX Shouldn't this be below with _log()?
import zLOG
zLOG.initialize()
def close_storages(self):
for name, storage in self.storages.items():
info("closing storage %r" % name)
try:
storage.close()
except: # Keep going
exception("failed to close storage %r" % name)
# Signal names
signames = None
def signame(sig):
"""Return a symbolic name for a signal.
Return "signal NNN" if there is no corresponding SIG name in the
signal module.
"""
if signames is None:
init_signames()
return signames.get(sig) or "signal %d" % sig
def init_signames():
global signames
signames = {}
for name, sig in signal.__dict__.items():
k_startswith = getattr(name, "startswith", None)
if k_startswith is None:
continue
if k_startswith("SIG") and not k_startswith("SIG_"):
signames[sig] = name
# Log messages with various severities.
# This uses zLOG, but the API is a simplified version of PEP 282
def critical(msg):
"""Log a critical message."""
_log(msg, zLOG.PANIC)
def error(msg):
"""Log an error message."""
_log(msg, zLOG.ERROR)
def exception(msg):
"""Log an exception (an error message with a traceback attached)."""
_log(msg, zLOG.ERROR, error=sys.exc_info())
def warn(msg):
"""Log a warning message."""
_log(msg, zLOG.PROBLEM)
def info(msg):
"""Log an informational message."""
_log(msg, zLOG.INFO)
def debug(msg):
"""Log a debugging message."""
_log(msg, zLOG.DEBUG)
def _log(msg, severity=zLOG.INFO, error=None):
"""Internal: generic logging function."""
zLOG.LOG("RUNSVR", severity, msg, "", error)
# Main program
def main(args=None):
options = ZEOOptions(args)
s = ZEOServer(options)
s.main()
if __name__ == "__main__":
main()
=== Zope/lib/python/ZEO/ClientCache.py 1.39.2.1 => 1.39.2.2 ===
--- Zope/lib/python/ZEO/ClientCache.py:1.39.2.1 Tue Oct 8 20:41:42 2002
+++ Zope/lib/python/ZEO/ClientCache.py Sun Nov 24 18:55:25 2002
@@ -111,9 +111,6 @@
import zLOG
from ZEO.ICache import ICache
-def log(msg, level=zLOG.INFO):
- zLOG.LOG("ZEC", level, msg)
-
magic='ZEC0'
class ClientCache:
@@ -122,11 +119,14 @@
def __init__(self, storage='1', size=20000000, client=None, var=None):
# Arguments:
- # storage -- storage name (used in persistent cache file names only)
+ # storage -- storage name (used in filenames and log messages)
# size -- size limit in bytes of both files together
# client -- if not None, use a persistent cache file and use this name
# var -- directory where to create persistent cache files
+ self._storage = storage
+ self._limit = size / 2
+
# Allocate locks:
L = allocate_lock()
self._acquire = L.acquire
@@ -182,10 +182,9 @@
f[0].write(magic)
current = 0
- log("%s: storage=%r, size=%r; file[%r]=%r" %
- (self.__class__.__name__, storage, size, current, p[current]))
+ self.log("%s: storage=%r, size=%r; file[%r]=%r" %
+ (self.__class__.__name__, storage, size, current, p[current]))
- self._limit = size / 2
self._current = current
self._setup_trace()
@@ -203,8 +202,8 @@
f = self._f
current = self._current
if f[not current] is not None:
- read_index(index, serial, f[not current], not current)
- self._pos = read_index(index, serial, f[current], current)
+ self.read_index(serial, not current)
+ self._pos = self.read_index(serial, current)
return serial.items()
finally:
@@ -240,15 +239,15 @@
f.seek(ap)
h = f.read(27)
if len(h) != 27:
- log("invalidate: short record for oid %16x "
- "at position %d in cache file %d"
- % (U64(oid), ap, p < 0))
+ self.log("invalidate: short record for oid %16x "
+ "at position %d in cache file %d"
+ % (U64(oid), ap, p < 0))
del self._index[oid]
return None
if h[:8] != oid:
- log("invalidate: oid mismatch: expected %16x read %16x "
- "at position %d in cache file %d"
- % (U64(oid), U64(h[:8]), ap, p < 0))
+ self.log("invalidate: oid mismatch: expected %16x read %16x "
+ "at position %d in cache file %d"
+ % (U64(oid), U64(h[:8]), ap, p < 0))
del self._index[oid]
return None
f.seek(ap+8) # Switch from reading to writing
@@ -281,9 +280,9 @@
else:
tlen = -1
if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen:
- log("load: bad record for oid %16x "
- "at position %d in cache file %d"
- % (U64(oid), ap, p < 0))
+ self.log("load: bad record for oid %16x "
+ "at position %d in cache file %d"
+ % (U64(oid), ap, p < 0))
del self._index[oid]
return None
@@ -452,9 +451,9 @@
else:
tlen = -1
if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen:
- log("modifiedInVersion: bad record for oid %16x "
- "at position %d in cache file %d"
- % (U64(oid), ap, p < 0))
+ self.log("modifiedInVersion: bad record for oid %16x "
+ "at position %d in cache file %d"
+ % (U64(oid), ap, p < 0))
del self._index[oid]
return None
@@ -481,7 +480,7 @@
current = not self._current
self._current = current
self._trace(0x70)
- log("flipping cache files. new current = %d" % current)
+ self.log("flipping cache files. new current = %d" % current)
# Delete the half of the index that's no longer valid
index = self._index
for oid in index.keys():
@@ -551,19 +550,21 @@
def _setup_trace(self):
# See if cache tracing is requested through $ZEO_CACHE_TRACE.
+ # A dash and the storage name are appended to get the filename.
# If not, or if we can't write to the trace file,
# disable tracing by setting self._trace to a dummy function.
self._tracefile = None
tfn = os.environ.get("ZEO_CACHE_TRACE")
if tfn:
+ tfn = tfn + "-" + self._storage
try:
self._tracefile = open(tfn, "ab")
self._trace(0x00)
except IOError, msg:
self._tracefile = None
- log("cannot write tracefile %s (%s)" % (tfn, msg))
+ self.log("cannot write tracefile %s (%s)" % (tfn, msg))
else:
- log("opened tracefile %s" % tfn)
+ self.log("opened tracefile %s" % tfn)
if self._tracefile is None:
def notrace(*args):
pass
@@ -587,85 +588,90 @@
oid,
serial))
-def read_index(index, serial, f, fileindex):
- seek = f.seek
- read = f.read
- pos = 4
- count = 0
+ def read_index(self, serial, fileindex):
+ index = self._index
+ f = self._f[fileindex]
+ seek = f.seek
+ read = f.read
+ pos = 4
+ count = 0
- while 1:
- f.seek(pos)
- h = read(27)
- if len(h) != 27:
- # An empty read is expected, anything else is suspect
- if h:
- rilog("truncated header", pos, fileindex)
- break
-
- if h[8] in 'vni':
- tlen, vlen, dlen = unpack(">iHi", h[9:19])
- else:
- tlen = -1
- if tlen <= 0 or vlen < 0 or dlen < 0 or vlen + dlen > tlen:
- rilog("invalid header data", pos, fileindex)
- break
-
- oid = h[:8]
-
- if h[8] == 'v' and vlen:
- seek(dlen+vlen, 1)
- vdlen = read(4)
- if len(vdlen) != 4:
- rilog("truncated record", pos, fileindex)
- break
- vdlen = unpack(">i", vdlen)[0]
- if vlen+dlen+43+vdlen != tlen:
- rilog("inconsistent lengths", pos, fileindex)
+ while 1:
+ f.seek(pos)
+ h = read(27)
+ if len(h) != 27:
+ # An empty read is expected, anything else is suspect
+ if h:
+ self.rilog("truncated header", pos, fileindex)
break
- seek(vdlen, 1)
- vs = read(8)
- if read(4) != h[9:13]:
- rilog("inconsistent tlen", pos, fileindex)
+
+ if h[8] in 'vni':
+ tlen, vlen, dlen = unpack(">iHi", h[9:19])
+ else:
+ tlen = -1
+ if tlen <= 0 or vlen < 0 or dlen < 0 or vlen + dlen > tlen:
+ self.rilog("invalid header data", pos, fileindex)
break
- else:
- if h[8] in 'vn' and vlen == 0:
- if dlen+31 != tlen:
- rilog("inconsistent nv lengths", pos, fileindex)
- seek(dlen, 1)
+
+ oid = h[:8]
+
+ if h[8] == 'v' and vlen:
+ seek(dlen+vlen, 1)
+ vdlen = read(4)
+ if len(vdlen) != 4:
+ self.rilog("truncated record", pos, fileindex)
+ break
+ vdlen = unpack(">i", vdlen)[0]
+ if vlen+dlen+43+vdlen != tlen:
+ self.rilog("inconsistent lengths", pos, fileindex)
+ break
+ seek(vdlen, 1)
+ vs = read(8)
if read(4) != h[9:13]:
- rilog("inconsistent nv tlen", pos, fileindex)
+ self.rilog("inconsistent tlen", pos, fileindex)
break
- vs = None
-
- if h[8] in 'vn':
- if fileindex:
- index[oid] = -pos
else:
- index[oid] = pos
- serial[oid] = h[-8:], vs
- else:
- if serial.has_key(oid):
- # We have a record for this oid, but it was invalidated!
- del serial[oid]
- del index[oid]
-
-
- pos = pos + tlen
- count += 1
-
- f.seek(pos)
- try:
- f.truncate()
- except:
- pass
-
- if count:
- log("read_index: cache file %d has %d records and %d bytes"
- % (fileindex, count, pos))
-
- return pos
-
-def rilog(msg, pos, fileindex):
- # Helper to log messages from read_index
- log("read_index: %s at position %d in cache file %d"
- % (msg, pos, fileindex))
+ if h[8] in 'vn' and vlen == 0:
+ if dlen+31 != tlen:
+ self.rilog("inconsistent nv lengths", pos, fileindex)
+ seek(dlen, 1)
+ if read(4) != h[9:13]:
+ self.rilog("inconsistent nv tlen", pos, fileindex)
+ break
+ vs = None
+
+ if h[8] in 'vn':
+ if fileindex:
+ index[oid] = -pos
+ else:
+ index[oid] = pos
+ serial[oid] = h[-8:], vs
+ else:
+ if serial.has_key(oid):
+ # We have a record for this oid, but it was invalidated!
+ del serial[oid]
+ del index[oid]
+
+
+ pos = pos + tlen
+ count += 1
+
+ f.seek(pos)
+ try:
+ f.truncate()
+ except:
+ pass
+
+ if count:
+ self.log("read_index: cache file %d has %d records and %d bytes"
+ % (fileindex, count, pos))
+
+ return pos
+
+ def rilog(self, msg, pos, fileindex):
+ # Helper to log messages from read_index
+ self.log("read_index: %s at position %d in cache file %d"
+ % (msg, pos, fileindex))
+
+ def log(self, msg, level=zLOG.INFO):
+ zLOG.LOG("ZEC:%s" % self._storage, level, msg)
=== Zope/lib/python/ZEO/ClientStorage.py 1.76.2.1 => 1.76.2.2 ===
--- Zope/lib/python/ZEO/ClientStorage.py:1.76.2.1 Tue Oct 8 20:41:42 2002
+++ Zope/lib/python/ZEO/ClientStorage.py Sun Nov 24 18:55:25 2002
@@ -28,9 +28,11 @@
import cPickle
import os
+import socket
import tempfile
import threading
import time
+import types
from ZEO import ClientCache, ServerStub
from ZEO.TransactionBuffer import TransactionBuffer
@@ -204,6 +206,8 @@
self._storage = storage
self._read_only_fallback = read_only_fallback
self._connection = None
+ # _server_addr is used by sortKey()
+ self._server_addr = None
self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client',
'supportsUndo':0, 'supportsVersions': 0,
@@ -339,6 +343,7 @@
log2(INFO, "Reconnected to storage")
else:
log2(INFO, "Connected to storage")
+ self.set_server_addr(conn.get_addr())
stub = self.StorageServerStubClass(conn)
self._oids = []
self._info.update(stub.get_info())
@@ -350,6 +355,33 @@
self._connection = conn
self._server = stub
+ def set_server_addr(self, addr):
+ # Normalize server address and convert to string
+ if isinstance(addr, types.StringType):
+ self._server_addr = addr
+ else:
+ assert isinstance(addr, types.TupleType)
+ # If the server is on a remote host, we need to guarantee
+ # that all clients used the same name for the server. If
+ # they don't, the sortKey() may be different for each client.
+ # The best solution seems to be the official name reported
+ # by gethostbyaddr().
+ host = addr[0]
+ try:
+ canonical, aliases, addrs = socket.gethostbyaddr(host)
+ except socket.error, err:
+ log2(BLATHER, "Error resoving host: %s (%s)" % (host, err))
+ canonical = host
+ self._server_addr = str((canonical, addr[1]))
+
+ def sortKey(self):
+ # If the client isn't connected to anything, it can't have a
+ # valid sortKey(). Raise an error to stop the transaction early.
+ if self._server_addr is None:
+ raise ClientDisconnected
+ else:
+ return self._server_addr
+
def verify_cache(self, server):
"""Internal routine called to verify the cache."""
# XXX beginZeoVerify ends up calling back to beginVerify() below.
@@ -398,6 +430,18 @@
"""Storage API: an approximate size of the database, in bytes."""
return self._info['size']
+ def getExtensionMethods(self):
+ """getExtensionMethods
+
+ This returns a dictionary whose keys are names of extra methods
+ provided by this storage. Storage proxies (such as ZEO) should
+ call this method to determine the extra methods that they need
+ to proxy in addition to the standard storage methods.
+ Dictionary values should be None; this will be a handy place
+ for extra marshalling information, should we need it
+ """
+ return self._info['extensionMethods']
+
def supportsUndo(self):
"""Storage API: return whether we support undo."""
return self._info['supportsUndo']
@@ -465,6 +509,12 @@
"""
return self._server.history(oid, version, length)
+ def __getattr__(self, name):
+ if self.getExtensionMethods().has_key(name):
+ return self._server.extensionMethod(name)
+ else:
+ raise AttributeError(name)
+
def loadSerial(self, oid, serial):
"""Storage API: load a historical revision of an object."""
return self._server.loadSerial(oid, serial)
@@ -604,11 +654,15 @@
"""Internal helper to end a transaction."""
# the right way to set self._transaction to None
# calls notify() on _tpc_cond in case there are waiting threads
+ self._ltid = self._serial
self._tpc_cond.acquire()
self._transaction = None
self._tpc_cond.notify()
self._tpc_cond.release()
+ def lastTransaction(self):
+ return self._ltid
+
def tpc_abort(self, transaction):
"""Storage API: abort a transaction."""
if transaction is not self._transaction:
@@ -792,3 +846,5 @@
invalidate = invalidateVerify
end = endVerify
Invalidate = invalidateTrans
+
+
=== Zope/lib/python/ZEO/README.txt 1.3.8.1 => 1.3.8.2 ===
--- Zope/lib/python/ZEO/README.txt:1.3.8.1 Tue Oct 8 20:41:42 2002
+++ Zope/lib/python/ZEO/README.txt Sun Nov 24 18:55:25 2002
@@ -59,11 +59,13 @@
----------------------
To enable cache tracing, set the environment variable ZEO_CACHE_TRACE
-to the name of a file to which the ZEO client process can write. If
-the file doesn't exist, the ZEO will try to create it. If there are
-problems with the file, a log message is written to the standard Zope
-log file. To start or stop tracing, the ZEO client process (typically
-a Zope application server) must be restarted.
+to the name of a file to which the ZEO client process can write. ZEO
+will append a hyphen and the storage name to the filename, to
+distinguish different storages. If the file doesn't exist, the ZEO
+will try to create it. If there are problems with the file, a log
+message is written to the standard Zope log file. To start or stop
+tracing, the ZEO client process (typically a Zope application server)
+must be restarted.
The trace file can grow pretty quickly; on a moderately loaded server,
we observed it growing by 5 MB per hour. The file consists of binary
=== Zope/lib/python/ZEO/ServerStub.py 1.9.4.1 => 1.9.4.2 ===
--- Zope/lib/python/ZEO/ServerStub.py:1.9.4.1 Tue Oct 8 20:41:42 2002
+++ Zope/lib/python/ZEO/ServerStub.py Sun Nov 24 18:55:25 2002
@@ -33,6 +33,9 @@
"""
self.rpc = rpc
+ def extensionMethod(self, name):
+ return ExtensionMethodWrapper(self.rpc, name).call
+
def _update(self):
"""Handle pending incoming messages.
@@ -137,3 +140,10 @@
return self.rpc.call('versions')
else:
return self.rpc.call('versions', max)
+
+class ExtensionMethodWrapper:
+ def __init__(self, rpc, name):
+ self.rpc = rpc
+ self.name = name
+ def call(self, *a, **kwa):
+ return apply(self.rpc.call, (self.name,)+a, kwa)
=== Zope/lib/python/ZEO/StorageServer.py 1.76.2.1 => 1.76.2.2 ===
--- Zope/lib/python/ZEO/StorageServer.py:1.76.2.1 Tue Oct 8 20:41:42 2002
+++ Zope/lib/python/ZEO/StorageServer.py Sun Nov 24 18:55:25 2002
@@ -25,6 +25,7 @@
import os
import sys
import threading
+import time
from ZEO import ClientStub
from ZEO.CommitLog import CommitLog
@@ -205,6 +206,7 @@
def __init__(self, server, read_only=0):
self.server = server
+ self.connection = None
self.client = None
self.storage = None
self.storage_id = "uninitialized"
@@ -212,6 +214,7 @@
self.read_only = read_only
def notifyConnected(self, conn):
+ self.connection = conn # For restart_other() below
self.client = self.ClientStorageStubClass(conn)
def notifyDisconnected(self):
@@ -244,6 +247,18 @@
self.load = self.storage.load
self.loadSerial = self.storage.loadSerial
self.modifiedInVersion = self.storage.modifiedInVersion
+ try:
+ fn = self.storage.getExtensionMethods
+ except AttributeError:
+ # We must be running with a ZODB which
+ # predates adding getExtensionMethods to
+ # BaseStorage. Eventually this try/except
+ # can be removed
+ pass
+ else:
+ for name in fn().keys():
+ if not hasattr(self,name):
+ setattr(self, name, getattr(self.storage, name))
def check_tid(self, tid, exc=None):
if self.read_only:
@@ -295,6 +310,7 @@
'supportsVersions': self.storage.supportsVersions(),
'supportsTransactionalUndo':
self.storage.supportsTransactionalUndo(),
+ 'extensionMethods': self.getExtensionMethods(),
}
def get_size_info(self):
@@ -302,6 +318,14 @@
'size': self.storage.getSize(),
}
+ def getExtensionMethods(self):
+ try:
+ e = self.storage.getExtensionMethods
+ except AttributeError:
+ return {}
+ else:
+ return e()
+
def zeoLoad(self, oid):
v = self.storage.modifiedInVersion(oid)
if v:
@@ -390,12 +414,8 @@
" requests from one client.")
# (This doesn't require a lock because we're using asyncore)
- if self.storage._transaction is None:
- self.strategy = self.ImmediateCommitStrategyClass(self.storage,
- self.client)
- else:
- self.strategy = self.DelayedCommitStrategyClass(self.storage,
- self.wait)
+ self.strategy = self.DelayedCommitStrategyClass(self.storage,
+ self.wait)
t = Transaction()
t.id = id
@@ -472,8 +492,10 @@
"Clients waiting: %d." % len(self.storage._waiting))
return d
else:
- self.restart()
- return None
+ return self.restart()
+
+ def dontwait(self):
+ return self.restart()
def handle_waiting(self):
while self.storage._waiting:
@@ -495,7 +517,7 @@
except:
self.log("Unexpected error handling waiting transaction",
level=zLOG.WARNING, error=sys.exc_info())
- zeo_storage._conn.close()
+ zeo_storage.connection.close()
return 0
else:
return 1
@@ -508,6 +530,8 @@
resp = old_strategy.restart(self.strategy)
if delay is not None:
delay.reply(resp)
+ else:
+ return resp
# A ZEOStorage instance can use different strategies to commit a
# transaction. The current implementation uses different strategies
=== Zope/lib/python/ZEO/__init__.py 1.13.2.1 => 1.13.2.2 ===
--- Zope/lib/python/ZEO/__init__.py:1.13.2.1 Tue Oct 8 20:41:42 2002
+++ Zope/lib/python/ZEO/__init__.py Sun Nov 24 18:55:25 2002
@@ -21,4 +21,4 @@
"""
-version = "2.0b2"
+version = "2.0+"
=== Zope/lib/python/ZEO/simul.py 1.13.2.2 => 1.13.2.3 ===
--- Zope/lib/python/ZEO/simul.py:1.13.2.2 Sat Oct 26 15:51:48 2002
+++ Zope/lib/python/ZEO/simul.py Sun Nov 24 18:55:25 2002
@@ -14,7 +14,7 @@
##############################################################################
"""Cache simulation.
-Usage: simul.py [-bflyz] [-s size] tracefile
+Usage: simul.py [-bflyz] [-X] [-s size] tracefile
Use one of -b, -f, -l, -y or -z select the cache simulator:
-b: buddy system allocator
@@ -25,6 +25,8 @@
Options:
-s size: cache size in MB (default 20 MB)
+-X: enable heuristic checking for misaligned records: oids > 2**32
+ will be rejected; this requires the tracefile to be seekable
Note: the buddy system allocator rounds the cache size up to a power of 2
"""
@@ -43,8 +45,9 @@
MB = 1000*1000
cachelimit = 20*MB
simclass = ZEOCacheSimulation
+ heuristic = 0
try:
- opts, args = getopt.getopt(sys.argv[1:], "bflyzs:")
+ opts, args = getopt.getopt(sys.argv[1:], "bflyzs:X")
except getopt.error, msg:
usage(msg)
return 2
@@ -61,6 +64,8 @@
simclass = ZEOCacheSimulation
if o == '-s':
cachelimit = int(float(a)*MB)
+ if o == '-X':
+ heuristic = 1
if len(args) != 1:
usage("exactly one file argument required")
return 2
@@ -112,12 +117,18 @@
# Must be a misaligned record caused by a crash
##print "Skipping 8 bytes at offset", offset-8
continue
- r = f_read(16)
- if len(r) < 16:
+ oid = f_read(8)
+ if len(oid) < 8:
break
- offset += 16
+ if heuristic and oid[:4] != '\0\0\0\0':
+ f.seek(-8, 1)
+ continue
+ offset += 8
+ serial = f_read(8)
+ if len(serial) < 8:
+ break
+ offset += 8
records += 1
- oid, serial = struct_unpack(">8s8s", r)
# Decode the code
dlen, version, code, current = (code & 0x7fffff00,
code & 0x80,
=== Zope/lib/python/ZEO/start.py 1.47.2.2 => 1.47.2.3 ===
--- Zope/lib/python/ZEO/start.py:1.47.2.2 Sat Oct 26 15:51:48 2002
+++ Zope/lib/python/ZEO/start.py Sun Nov 24 18:55:26 2002
@@ -17,6 +17,8 @@
import sys, os, getopt
import types
+import errno
+import socket
def directory(p, n=1):
d = p
@@ -93,6 +95,8 @@
def main(argv):
me = argv[0]
sys.path.insert(0, directory(me, 2))
+ import zLOG
+ zLOG.initialize()
global LOG, INFO, ERROR
from zLOG import LOG, INFO, WARNING, ERROR, PANIC
@@ -118,7 +122,7 @@
-D -- Run in debug mode
- -d -- Set STUPD_LOG_SEVERITY to -300
+ -d -- Set STUPID_LOG_SEVERITY to -300
-U -- Unix-domain socket file to listen on
@@ -201,7 +205,7 @@
if args:
if len(args) > 1:
print usage
- print 'Unrecognizd arguments: ', " ".join(args[1:])
+ print 'Unrecognized arguments: ', " ".join(args[1:])
sys.exit(1)
fs = args[0]
@@ -209,6 +213,7 @@
os.environ['Z_DEBUG_MODE'] = '1'
if detailed:
os.environ['STUPID_LOG_SEVERITY'] = '-300'
+ zLOG.initialize()
set_uid(UID)
@@ -306,17 +311,18 @@
def rotate_logs():
import zLOG
- # There hasn't been a clear way to reinitialize the MinimalLogger.
- # I'll checkin the public initialize() method soon, but also try some
- # other strategies for older Zope installs :-(.
init = getattr(zLOG, 'initialize', None)
if init is not None:
init()
return
# This will work if the minimal logger is in use, but not if some
- # other logger is active.
- import zLOG.MinimalLogger
- zLOG.MinimalLogger._log.initialize()
+ # other logger is active. MinimalLogger exists only in Zopes
+ # pre-2.7.
+ try:
+ import zLOG.MinimalLogger
+ zLOG.MinimalLogger._log.initialize()
+ except ImportError:
+ pass
def rotate_logs_handler(signum, frame):
rotate_logs()
=== Zope/lib/python/ZEO/stats.py 1.18.2.1 => 1.18.2.2 ===
--- Zope/lib/python/ZEO/stats.py:1.18.2.1 Tue Oct 8 20:41:42 2002
+++ Zope/lib/python/ZEO/stats.py Sun Nov 24 18:55:26 2002
@@ -14,13 +14,15 @@
##############################################################################
"""Trace file statistics analyzer.
-Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] tracefile
+Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
+-X: enable heuristic checking for misaligned records: oids > 2**32
+ will be rejected; this requires the tracefile to be seekable
"""
"""File format:
@@ -67,8 +69,9 @@
print_size_histogram = 0
print_histogram = 0
interval = 900 # Every 15 minutes
+ heuristic = 0
try:
- opts, args = getopt.getopt(sys.argv[1:], "hi:qsSv")
+ opts, args = getopt.getopt(sys.argv[1:], "hi:qsSvX")
except getopt.error, msg:
usage(msg)
return 2
@@ -90,6 +93,8 @@
dostats = 0
if o == "-v":
verbose = 1
+ if o == '-X':
+ heuristic = 1
if len(args) != 1:
usage("exactly one file argument required")
return 2
@@ -148,14 +153,24 @@
if ts == 0:
# Must be a misaligned record caused by a crash
if not quiet:
- print "Skipping 8 bytes at offset", offset-8
+ print "Skipping 8 bytes at offset", offset-8,
+ print repr(r)
continue
- r = f_read(16)
- if len(r) < 16:
+ oid = f_read(8)
+ if len(oid) < 8:
break
- offset += 16
+ if heuristic and oid[:4] != '\0\0\0\0':
+ # Heuristic for severe data corruption
+ print "Seeking back over bad oid at offset", offset,
+ print repr(r)
+ f.seek(-8, 1)
+ continue
+ offset += 8
+ serial = f_read(8)
+ if len(serial) < 8:
+ break
+ offset += 8
records += 1
- oid, serial = struct_unpack(">8s8s", r)
if t0 is None:
t0 = ts
thisinterval = t0 / interval
=== Zope/lib/python/ZEO/version.txt 1.4.2.1 => 1.4.2.2 ===
--- Zope/lib/python/ZEO/version.txt:1.4.2.1 Tue Oct 8 20:41:42 2002
+++ Zope/lib/python/ZEO/version.txt Sun Nov 24 18:55:26 2002
@@ -1,2 +1 @@
-2.0b2
-
+2.0+