[Zodb-checkins] SVN: ZODB/trunk/src/ Rewrote zeopack:
Jim Fulton
jim at zope.com
Tue Mar 24 17:07:35 EDT 2009
Log message for revision 98338:
Rewrote zeopack:
The zeopack script has gotten a numner of improvements:
- Simplified command-line interface. (The old interface is still
supported, except that support for ZEO version 1 servers has been
dropped.)
- Multiple storages can be packed in sequence.
- This simplifies pack scheduling on servers serving multiple
databases.
- All storages are packed to the same time.
- You can now specify a time of day to pack to.
- The script will now time out if it can't connect to s storage in
60 seconds.
- It now has a test. :)
Changed:
U ZODB/trunk/src/CHANGES.txt
A ZODB/trunk/src/ZEO/scripts/tests.py
U ZODB/trunk/src/ZEO/scripts/zeopack.py
A ZODB/trunk/src/ZEO/scripts/zeopack.test
-=-
Modified: ZODB/trunk/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt 2009-03-24 20:14:09 UTC (rev 98337)
+++ ZODB/trunk/src/CHANGES.txt 2009-03-24 21:07:34 UTC (rev 98338)
@@ -8,6 +8,30 @@
New Features
------------
+3.9.0a13 (2009-??-??)
+=====================
+
+New Features
+------------
+
+- The zeopack script has gotten a numner of improvements:
+
+ - Simplified command-line interface. (The old interface is still
+ supported, except that support for ZEO version 1 servers has been
+ dropped.)
+
+ - Multiple storages can be packed in sequence.
+
+ - This simplifies pack scheduling on servers serving multiple
+ databases.
+
+ - All storages are packed to the same time.
+
+ - You can now specify a time of day to pack to.
+
+ - The script will now time out if it can't connect to s storage in
+ 60 seconds.
+
- The connection now estimates the object size based on its pickle size
and informs the cache about size changes.
@@ -19,15 +43,6 @@
There are corresponding methods to read and set the new configuration
parameters.
- XXX There are known issues with this implementation that need to be
- sorted out before it is "released".
-
-- Moved a blob related logging message to debug level, for the case of
- everything working as intended.
-
-- Changed the url in the package metadata to point to PyPi. Also removed
- references to download.zope.org.
-
3.9.0a12 (2009-02-26)
=====================
Added: ZODB/trunk/src/ZEO/scripts/tests.py
===================================================================
--- ZODB/trunk/src/ZEO/scripts/tests.py (rev 0)
+++ ZODB/trunk/src/ZEO/scripts/tests.py 2009-03-24 21:07:34 UTC (rev 98338)
@@ -0,0 +1,25 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""XXX short summary goes here.
+
+$Id$
+"""
+import unittest
+from zope.testing import doctest
+
+def test_suite():
+ return unittest.TestSuite((
+ doctest.DocFileSuite('zeopack.test'),
+ ))
+
Property changes on: ZODB/trunk/src/ZEO/scripts/tests.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Modified: ZODB/trunk/src/ZEO/scripts/zeopack.py
===================================================================
--- ZODB/trunk/src/ZEO/scripts/zeopack.py 2009-03-24 20:14:09 UTC (rev 98337)
+++ ZODB/trunk/src/ZEO/scripts/zeopack.py 2009-03-24 21:07:34 UTC (rev 98338)
@@ -1,123 +1,182 @@
#!/usr/bin/env python2.3
-"""Connect to a ZEO server and ask it to pack.
+import logging
+import optparse
+import socket
+import sys
+import time
+import traceback
+import ZEO.ClientStorage
-Usage: zeopack.py [options]
+usage = """Usage: %prog [options] [servers]
-Options:
+Pack one or more storages hosted by ZEO servers.
- -p port -- port to connect to
+The positional arguments specify 0 or more tcp servers to pack, where
+each is of the form:
- -h host -- host to connect to (default is current host)
+ host:port[:name]
- -U path -- Unix-domain socket to connect to
+"""
- -S name -- storage name (default is '1')
+WAIT = 10 # wait no more than 10 seconds for client to connect
- -d days -- pack objects more than days old
+def _main(args=None, prog=None):
+ if args is None:
+ args = sys.argv[1:]
- -1 -- Connect to a ZEO 1 server
+ parser = optparse.OptionParser(usage, prog=prog)
- -W -- wait for server to come up. Normally the script tries to
- connect for 10 seconds, then exits with an error. The -W
- option is only supported with ZEO 1.
+ parser.add_option(
+ "-d", "--days", dest="days", type='int', default=0,
+ help=("Pack objects that are older than this number of days")
+ )
-You must specify either -p and -h or -U.
-"""
+ parser.add_option(
+ "-t", "--time", dest="time",
+ help=("Time of day to pack to of the form: HH[:MM[:SS]]. "
+ "Defaults to current time.")
+ )
-import getopt
-import socket
-import sys
-import time
+ parser.add_option(
+ "-u", "--unix", dest="unix_sockets", action="append",
+ help=("A unix-domain-socket server to connect to, of the form: "
+ "path[:name]")
+ )
-from ZEO.ClientStorage import ClientStorage
+ parser.remove_option('-h')
+ parser.add_option(
+ "-h", dest="host",
+ help=("Deprecated: "
+ "Used with the -p and -S options, specified the host to "
+ "connect to.")
+ )
-WAIT = 10 # wait no more than 10 seconds for client to connect
+ parser.add_option(
+ "-p", type="int", dest="port",
+ help=("Deprecated: "
+ "Used with the -h and -S options, specifies "
+ "the port to connect to.")
+ )
-def connect(storage):
- # The connect-on-startup logic that ZEO provides isn't too useful
- # for this script. We'd like to client to attempt to startup, but
- # fail if it can't get through to the server after a reasonable
- # amount of time. There's no external support for this, so we'll
- # expose the ZEO 1.0 internals. (consenting adults only)
- t0 = time.time()
- while t0 + WAIT > time.time():
- storage._call.connect()
- if storage._connected:
- return
- raise RuntimeError("Unable to connect to ZEO server")
+ parser.add_option(
+ "-S", dest="name", default='1',
+ help=("Deprecated: Used with the -h and -p, options, or with the "
+ "-U option specified the storage name to use. Defaults to 1.")
+ )
-def pack1(addr, storage, days, wait):
- cs = ClientStorage(addr, storage=storage,
- wait_for_server_on_startup=wait)
- if wait:
- # _startup() is an artifact of the way ZEO 1.0 works. The
- # ClientStorage doesn't get fully initialized until registerDB()
- # is called. The only thing we care about, though, is that
- # registerDB() calls _startup().
- cs._startup()
- else:
- connect(cs)
- cs.invalidator = None
- cs.pack(wait=1, days=days)
- cs.close()
+ parser.add_option(
+ "-U", dest="unix",
+ help=("Deprecated: Used with the -S option, "
+ "Unix-domain socket to connect to.")
+ )
-def pack2(addr, storage, days):
- cs = ClientStorage(addr, storage=storage, wait=1, read_only=1)
- cs.pack(wait=1, days=days)
- cs.close()
+ if not args:
+ parser.print_help()
+ return
-def usage(exit=1):
- print __doc__
- print " ".join(sys.argv)
- sys.exit(exit)
+ def error(message):
+ sys.stderr.write("Error:\n%s\n" % message)
+ sys.exit(1)
-def main():
- host = None
- port = None
- unix = None
- storage = '1'
- days = 0
- wait = 0
- zeoversion = 2
- try:
- opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:d:W1')
- for o, a in opts:
- if o == '-p':
- port = int(a)
- elif o == '-h':
- host = a
- elif o == '-U':
- unix = a
- elif o == '-S':
- storage = a
- elif o == '-d':
- days = int(a)
- elif o == '-W':
- wait = 1
- elif o == '-1':
- zeoversion = 1
- except Exception, err:
- print err
- usage()
+ options, args = parser.parse_args(args)
- if unix is not None:
- addr = unix
- else:
- if host is None:
- host = socket.gethostname()
- if port is None:
- usage()
- addr = host, port
+ packt = time.time()
+ if options.time:
+ time_ = map(int, options.time.split(':'))
+ if len(time_) == 1:
+ time_ += (0, 0)
+ elif len(time_) == 2:
+ time_ += (0,)
+ elif len(time_) > 3:
+ error("Invalid time value: %r" % options.time)
- if zeoversion == 1:
- pack1(addr, storage, days, wait)
- else:
- pack2(addr, storage, days)
+ packt = time.localtime(packt)
+ packt = time.mktime(packt[:3]+tuple(time_)+packt[6:])
+ packt -= options.days * 86400
+
+ servers = []
+
+ if options.host:
+ if not options.port:
+ error("If host (-h) is specified then a port (-p) must be "
+ "specified as well.")
+ servers.append(((options.host, options.port), options.name))
+ elif options.port:
+ error("If port (-p) is specified then a host (-h) must be "
+ "specified as well.")
+
+ if options.unix:
+ servers.append((options.unix, options.name))
+
+ for server in args:
+ data = server.split(':')
+ if len(data) in (2, 3):
+ host = data[0]
+ try:
+ port = int(data[1])
+ except ValueError:
+ error("Invalid port in server specification: %r" % server)
+ addr = host, port
+ if len(data) == 2:
+ name = '1'
+ else:
+ name = data[2]
+ else:
+ error("Invalid server specification: %r" % server)
+
+ servers.append((addr, name))
+
+ for server in options.unix_sockets or ():
+ data = server.split(':')
+ if len(data) == 1:
+ addr = data[0]
+ name = '1'
+ elif len(data) == 2:
+ addr = data[0]
+ name = data[1]
+ else:
+ error("Invalid server specification: %r" % server)
+
+ servers.append((addr, name))
+
+ if not servers:
+ error("No servers specified.")
+
+ for addr, name in servers:
+ try:
+ cs = ZEO.ClientStorage.ClientStorage(
+ addr, storage=name, wait=False, read_only=1)
+ for i in range(60):
+ if cs.is_connected():
+ break
+ time.sleep(1)
+ else:
+ sys.stderr.write("Couldn't connect to: %r\n"
+ % ((addr, name), ))
+ cs.close()
+ continue
+ cs.pack(packt, wait=True)
+ cs.close()
+ except:
+ traceback.print_exception(*(sys.exc_info()+(99, sys.stderr)))
+ error("Error packing storage %s in %r" % (name, addr))
+
+def main(*args):
+ root_logger = logging.getLogger()
+ old_level = root_logger.getEffectiveLevel()
+ logging.getLogger().setLevel(logging.WARNING)
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setFormatter(logging.Formatter(
+ "%(name)s %(levelname)s %(message)s"))
+ logging.getLogger().addHandler(handler)
+ try:
+ _main(*args)
+ finally:
+ logging.getLogger().setLevel(old_level)
+ logging.getLogger().removeHandler(handler)
+
if __name__ == "__main__":
- try:
- main()
- except Exception, err:
- print err
- sys.exit(1)
+ main()
+
Added: ZODB/trunk/src/ZEO/scripts/zeopack.test
===================================================================
--- ZODB/trunk/src/ZEO/scripts/zeopack.test (rev 0)
+++ ZODB/trunk/src/ZEO/scripts/zeopack.test 2009-03-24 21:07:34 UTC (rev 98338)
@@ -0,0 +1,269 @@
+zeopack
+=======
+
+The zeopack script can be used to pack one or more storages. It uses
+ClientStorage to do this. To test it's behavior, we'll replace the
+normal ClientStorage with a fake one that echos information we'll want
+for our test:
+
+ >>> class ClientStorage:
+ ... connect_wait = 0
+ ... def __init__(self, *args, **kw):
+ ... if args[0] == 'bad':
+ ... import logging
+ ... logging.getLogger('test.ClientStorage').error(
+ ... "I hate this address, %r", args[0])
+ ... raise ValueError("Bad address")
+ ... print "ClientStorage(%s %s)" % (
+ ... repr(args)[1:-1],
+ ... ', '.join("%s=%r" % i for i in sorted(kw.items())),
+ ... )
+ ... def pack(self, *args, **kw):
+ ... print "pack(%s %s)" % (
+ ... repr(args)[1:-1],
+ ... ', '.join("%s=%r" % i for i in sorted(kw.items())),
+ ... )
+ ... def is_connected(self):
+ ... self.connect_wait -= 1
+ ... print 'is_connected', self.connect_wait < 0
+ ... return self.connect_wait < 0
+ ... def close(self):
+ ... print "close()"
+
+ >>> import ZEO
+ >>> ClientStorage_orig = ZEO.ClientStorage.ClientStorage
+ >>> ZEO.ClientStorage.ClientStorage = ClientStorage
+
+Now, we're ready to try the script:
+
+ >>> from ZEO.scripts.zeopack import main
+
+If we call it with no arguments, we get help:
+
+>>> main([], 'zeopack')
+Usage: zeopack [options] [servers]
+<BLANKLINE>
+Pack one or more storages hosted by ZEO servers.
+<BLANKLINE>
+The positional arguments specify 0 or more tcp servers to pack, where
+each is of the form:
+<BLANKLINE>
+ host:port[:name]
+<BLANKLINE>
+<BLANKLINE>
+<BLANKLINE>
+Options:
+ -d DAYS, --days=DAYS Pack objects that are older than this number of days
+ -t TIME, --time=TIME Time of day to pack to of the form: HH[:MM[:SS]].
+ Defaults to current time.
+ -u UNIX_SOCKETS, --unix=UNIX_SOCKETS
+ A unix-domain-socket server to connect to, of the
+ form: path[:name]
+ -h HOST Deprecated: Used with the -p and -S options, specified
+ the host to connect to.
+ -p PORT Deprecated: Used with the -h and -S options, specifies
+ the port to connect to.
+ -S NAME Deprecated: Used with the -h and -p, options, or with
+ the -U option specified the storage name to use.
+ Defaults to 1.
+ -U UNIX Deprecated: Used with the -S option, Unix-domain
+ socket to connect to.
+
+Since packing involved time, we'd better have our way with it:
+
+ >>> import time
+ >>> time_orig = time.time
+ >>> time.time = lambda : 1237906517.0
+ >>> import os
+ >>> oldtz = os.environ.get('TZ')
+ >>> os.environ['TZ'] = 'PST+07'
+ >>> sleep_orig = time.sleep
+ >>> def sleep(t):
+ ... print 'sleep(%r)' % t
+ >>> time.sleep = sleep
+
+Normally, we pass one or more TCP server specifications:
+
+ >>> main(["host1:8100", "host1:8100:2"])
+ ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
+ is_connected True
+ pack(1237906517.0, wait=True)
+ close()
+ ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+ is_connected True
+ pack(1237906517.0, wait=True)
+ close()
+
+We can also pass unix-domain-sockey servers using the -u option:
+
+ >>> main(["-ufoo", "-ubar:spam", "host1:8100", "host1:8100:2"])
+ ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
+ is_connected True
+ pack(1237906517.0, wait=True)
+ close()
+ ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+ is_connected True
+ pack(1237906517.0, wait=True)
+ close()
+ ClientStorage('foo', read_only=1, storage='1', wait=False)
+ is_connected True
+ pack(1237906517.0, wait=True)
+ close()
+ ClientStorage('bar', read_only=1, storage='spam', wait=False)
+ is_connected True
+ pack(1237906517.0, wait=True)
+ close()
+
+The -d option causes a pack time the given number of days earlier to
+be used:
+
+ >>> main(["-ufoo", "-ubar:spam", "-d3", "host1:8100", "host1:8100:2"])
+ ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
+ is_connected True
+ pack(1237647317.0, wait=True)
+ close()
+ ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+ is_connected True
+ pack(1237647317.0, wait=True)
+ close()
+ ClientStorage('foo', read_only=1, storage='1', wait=False)
+ is_connected True
+ pack(1237647317.0, wait=True)
+ close()
+ ClientStorage('bar', read_only=1, storage='spam', wait=False)
+ is_connected True
+ pack(1237647317.0, wait=True)
+ close()
+
+The -t option allows us to control the time of day:
+
+ >>> main(["-ufoo", "-d3", "-t1:30", "host1:8100:2"])
+ ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+ is_connected True
+ pack(1237624200.0, wait=True)
+ close()
+ ClientStorage('foo', read_only=1, storage='1', wait=False)
+ is_connected True
+ pack(1237624200.0, wait=True)
+ close()
+
+Connection timeout
+------------------
+
+The zeopack script tells ClientStorage not to wait for connections
+before returning from the constructor, but will time out after 60
+seconds of waiting for a connect.
+
+ >>> ClientStorage.connect_wait = 3
+ >>> main(["-d3", "-t1:30", "host1:8100:2"])
+ ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+ is_connected False
+ sleep(1)
+ is_connected False
+ sleep(1)
+ is_connected False
+ sleep(1)
+ is_connected True
+ pack(1237624200.0, wait=True)
+ close()
+
+ >>> def call_main(args):
+ ... import sys
+ ... old_stderr = sys.stderr
+ ... sys.stderr = sys.stdout
+ ... try:
+ ... try:
+ ... main(args)
+ ... except SystemExit, v:
+ ... print "Exited", v
+ ... finally:
+ ... sys.stderr = old_stderr
+
+ >>> ClientStorage.connect_wait = 999
+ >>> call_main(["-d3", "-t1:30", "host1:8100", "host1:8100:2"])
+ ... # doctest: +ELLIPSIS
+ ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
+ is_connected False
+ sleep(1)
+ ...
+ is_connected False
+ sleep(1)
+ Couldn't connect to: (('host1', 8100), '1')
+ close()
+ ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+ is_connected False
+ sleep(1)
+ ...
+ is_connected False
+ sleep(1)
+ Couldn't connect to: (('host1', 8100), '2')
+ close()
+
+ >>> ClientStorage.connect_wait = 0
+
+
+Legacy support
+--------------
+
+ >>> main(["-d3", "-h", "host1", "-p", "8100", "-S", "2"])
+ ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
+ is_connected True
+ pack(1237647317.0, wait=True)
+ close()
+
+ >>> main(["-d3", "-U", "foo/bar", "-S", "2"])
+ ClientStorage('foo/bar', read_only=1, storage='2', wait=False)
+ is_connected True
+ pack(1237647317.0, wait=True)
+ close()
+
+Error handling
+--------------
+
+ >>> call_main(["-d3"])
+ Error:
+ No servers specified.
+ Exited 1
+
+ >>> call_main(["-d3", "a"])
+ Error:
+ Invalid server specification: 'a'
+ Exited 1
+
+ >>> call_main(["-d3", "a:b:c:d"])
+ Error:
+ Invalid server specification: 'a:b:c:d'
+ Exited 1
+
+ >>> call_main(["-d3", "a:b:2"])
+ Error:
+ Invalid port in server specification: 'a:b:2'
+ Exited 1
+
+ >>> call_main(["-d3", "-u", "a:b:2"])
+ Error:
+ Invalid server specification: 'a:b:2'
+ Exited 1
+
+ >>> call_main(["-d3", "-u", "bad"]) # doctest: +ELLIPSIS
+ test.ClientStorage ERROR I hate this address, 'bad'
+ Traceback (most recent call last):
+ ...
+ ValueError: Bad address
+ Error:
+ Error packing storage 1 in 'bad'
+ Exited 1
+
+
+
+Note that in the previous example, the first line was output through logging.
+
+.. tear down
+
+ >>> ZEO.ClientStorage.ClientStorage = ClientStorage_orig
+ >>> time.time = time_orig
+ >>> time.sleep = sleep_orig
+ >>> if oldtz is None:
+ ... del os.environ['TZ']
+ ... else:
+ ... os.environ['TZ'] = oldtz
Property changes on: ZODB/trunk/src/ZEO/scripts/zeopack.test
___________________________________________________________________
Added: svn:eol-style
+ native
More information about the Zodb-checkins
mailing list