[Zope-Checkins] CVS: StandaloneZODB/Tools - timeiter.py:1.1.2.7

[Guido van Rossum]

Update of /cvs-repository/StandaloneZODB/Tools
In directory cvs.zope.org:/tmp/cvs-serv11964

Modified Files:
      Tag: Standby-branch
	timeiter.py 
Log Message:
- Make the regular output format look nicer.  The columnized output
  helped us realize the problem with the Spread 1000-message limit!
  The columnized output also contains the serial number in the source
  storage.

- Print some more stats at the end and format them nicer.

- Add zeo client storage (only usable as destination).

- Add null storage (destination only) -- useful to print the stats
  for a source storage.


=== StandaloneZODB/Tools/timeiter.py 1.1.2.6 => 1.1.2.7 ===
         t1 = time.time()
         if options.verbose > 0:
-            print 'Migration time:', t1-t0
+            print 'Migration time:          %8.3f' % (t1-t0)
     finally:
         # Done
         srcdb.close()
@@ -195,6 +195,8 @@
     largest_pickle = 0
     largest_txn_in_size = 0
     largest_txn_in_objects = 0
+    total_pickle_size = 0
+    total_object_count = 0
     # Ripped from BaseStorage.copyTransactionsFrom()
     ts = None
     ok = 1
@@ -235,7 +237,11 @@
         objects = 0
         size = 0
         t0 = time.time()
-        dstdb.tpc_begin(txn, tid, txn.status)
+        if options.dtype == "zeo":
+            # Hack: ZEO client storage only supports 1-arg tpc_begin
+            dstdb.tpc_begin(txn)
+        else:
+            dstdb.tpc_begin(txn, tid, txn.status)
         t1 = time.time()
         for r in txn:
             oid = r.oid
@@ -268,21 +274,29 @@
 	    largest_txn_in_objects = objects
 	if size > largest_txn_in_size:
 	    largest_txn_in_size = size
-        print >> outfp, utils.U64(tid), objects, size, t4-t0, \
-              t1-t0, t2-t1, t3-t2, t4-t3
+        print >> outfp, "%4d. %20d %6d %8d %6.4f %6.4f %6.4f %6.4f %6.4f" % (
+            skipper, utils.U64(tid), objects, size,
+            t4-t0, t1-t0, t2-t1, t3-t2, t4-t3)
+        total_pickle_size += size
+        total_object_count += objects
 
         if prof:
             prof.create_stats()
             fp = open('profile-%02d.txt' % (counter / 100), 'wb')
             marshal.dump(prof.stats, fp)
             fp.close()
-    print >> outfp, largest_pickle, largest_txn_in_size, largest_txn_in_objects
+    print >> outfp, "Largest pickle:          %8d" % largest_pickle
+    print >> outfp, "Largest transaction:     %8d" % largest_txn_in_size
+    print >> outfp, "Largest object count:    %8d" % largest_txn_in_objects
+    print >> outfp, "Total pickle size:       %8d" % total_pickle_size
+    print >> outfp, "Total object count:      %8d" % total_object_count
 
 
 
 # This cruft is necessary because otherwise, it's just too dang hard to get
 # the right arguments to the various constructors.  If you've got a better
 # idea, I'm all ears!
+# XXX What cruft?
 def make_fs(args, read_only=0):
     """Use a read/write FileStorage.
 
@@ -338,6 +352,36 @@
     p.start() # Start the Spread message receive loop
     return p
 
+def make_zeo_client(args):
+    """Use a ZEO client storage.  Destination only!
+
+    Required arguments:
+        host -- the host to connect to
+        port -- the port to connect to
+    """
+    if len(args) < 2:
+        error(2, 'zeo requires exactly 2 arguments: host port')
+    host, port = args[:2]
+    del args[:2]
+    try:
+        port = int(port)
+    except:
+        error(2, 'zeo port argument should be an int (%s)' % `args[1]`)
+    from ZEO.ClientStorage import ClientStorage
+    addr = host, port
+    storage = ClientStorage(addr, wait_for_server_on_startup=1)
+    return storage
+
+def make_null_client(args):
+    """Use a Null storage -- which doesn't store anything.  Destination only!
+
+    Takes no arguments.
+    """
+    class NullStorage:
+        def noop(self, *args): pass
+        tpc_begin = store = tpc_vote = tpc_finish = tpc_abort = close = noop
+    return NullStorage()
+        
 
 # This maps case-insensitive names against factory functions.  The latter must
 # take a single argument which is the left-over command line, and return a
@@ -347,6 +391,8 @@
     'file'   : make_fs,
     'rofile' : make_rofs,
     'primary': make_primary,
+    'zeo'    : make_zeo_client,
+    'null'   : make_null_client,
 ##    'bdb'    : make_berkeley,
     }