[Zope-Checkins] CVS: Zope/lib/python/ZODB - ActivityMonitor.py:1.1.2.1 Connection.py:1.66.12.1 DB.py:1.41.12.1

Shane Hathaway shane@cvs.zope.org
Fri, 31 May 2002 16:23:23 -0400


Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv30758/lib/python/ZODB

Modified Files:
      Tag: shane-activity-monitoring-branch
	Connection.py DB.py 
Added Files:
      Tag: shane-activity-monitoring-branch
	ActivityMonitor.py 
Log Message:
Added an activity monitor to the Database control panel.  It is a bar graph
that shows how many objects have been loaded and stored recently.  It's
good for:

- discovering excessive loading, meaning you should raise the cache size;

- discovering excessive storing, meaning your application may be writing too
  often; and

- getting a general handle on the activity of your database.


=== Added File Zope/lib/python/ZODB/ActivityMonitor.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
# 
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
# 
##############################################################################
"""ZODB transfer activity monitoring

$Id: ActivityMonitor.py,v 1.1.2.1 2002/05/31 20:23:22 shane Exp $"""
__version__='$Revision: 1.1.2.1 $'[11:-2]

import time


class ActivityMonitor:
    """ZODB load/store activity monitor

    This simple implementation just keeps a log and iterates over the log for
    every call to getActivityAnalysis().

    It assumes that log entries are added in sequence, which is only
    guaranteed because DB.py holds a lock when calling the closedConnection()
    method.
    """

    def __init__(self, history_length=3600):
        self.history_length = history_length  # Number of seconds
        self.log = []                     # [(time, loads, stores)]

    def closedConnection(self, conn):
        log = self.log
        now = time.time()
        loads, stores = conn.getTransferCounts(1)
        log.append((now, loads, stores))
        self.trim(now)

    def trim(self, now):
        log = self.log
        cutoff = now - self.history_length
        n = 0
        loglen = len(log)
        while n < loglen and log[n][0] < cutoff:
            n = n + 1
        if n:
            del log[:n]

    def setHistoryLength(self, history_length):
        self.history_length = history_length
        self.trim(time.time())

    def getHistoryLength(self):
        return self.history_length

    def getActivityAnalysis(self, start=0, end=0, divisions=10):
        res = []
        log = self.log
        now = time.time()
        if start == 0:
            start = now - self.history_length
        if end == 0:
            end = now
        for n in range(divisions):
            res.append({
                'start': start + (end - start) * n / divisions,
                'end': start + (end - start) * (n + 1) / divisions,
                'loads': 0,
                'stores': 0,
                })

        div = res[0]
        div_start = div['start']
        div_end = div['end']
        div_index = 0
        total_loads = 0
        total_stores = 0
        for t, loads, stores in self.log:
            if t < start:
                # We could use a binary search to find the start.
                continue
            elif t > end:
                # We could use a binary search to find the end also.
                break
            while t > div_end:
                div['loads'] = total_loads
                div['stores'] = total_stores
                total_loads = 0
                total_stores = 0
                div_index = div_index + 1
                if div_index < divisions:
                    div = res[div_index]
                    div_start = div['start']
                    div_end = div['end']
            total_loads = total_loads + loads
            total_stores = total_stores + stores

        div['stores'] = div['stores'] + total_stores
        div['loads'] = div['loads'] + total_loads

        return res



=== Zope/lib/python/ZODB/Connection.py 1.66 => 1.66.12.1 ===
         self._committed=[]
         self._code_timestamp = global_code_timestamp
+        self._load_count = 0   # Number of objects unghosted
+        self._store_count = 0  # Number of objects stored
 
     def _cache_items(self):
         # find all items on the lru list
@@ -383,6 +385,7 @@
             dump(state)
             p=file(1)
             s=dbstore(oid,serial,p,version,transaction)
+            self._store_count = self._store_count + 1
             # Put the object in the cache before handling the
             # response, just in case the response contains the
             # serial number for a newly created object
@@ -486,6 +489,7 @@
 
         try:
             p, serial = self._storage.load(oid, self._version)
+            self._load_count = self._load_count + 1
 
             # XXX this is quite conservative!
             # We need, however, to avoid reading data from a transaction
@@ -691,6 +695,17 @@
 
     def getDebugInfo(self): return self._debug_info
     def setDebugInfo(self, *args): self._debug_info=self._debug_info+args
+
+    def getTransferCounts(self, clear=0):
+        """Returns the number of objects loaded and stored.
+
+        Set the clear argument to reset the counters.
+        """
+        res = (self._load_count, self._store_count)
+        if clear:
+            self._load_count = 0
+            self._store_count = 0
+        return res
 
 
     ######################################################################


=== Zope/lib/python/ZODB/DB.py 1.41 => 1.41.12.1 ===
     of managing objects is done by the connections.
     """
-    klass = Connection
+    klass = Connection  # Class to use for connections
+    _activity_monitor = None
 
     def __init__(self, storage,
                  pool_size=7,
@@ -124,6 +125,9 @@
         """Return a connection to the pool"""
         self._a()
         try:
+            am = self._activity_monitor
+            if am is not None:
+                am.closedConnection(connection)
             version=connection._version
             pools,pooll=self._pools
             pool, allocated, pool_lock = pools[version]
@@ -486,6 +490,9 @@
                     })
         return r
         
+    def getActivityMonitor(self):
+        return self._activity_monitor
+    
     def pack(self, t=None, days=0):
         if t is None: t=time()
         t=t-(days*86400)
@@ -515,7 +522,10 @@
 
     def setPoolSize(self, v):
         self._pool_size=v
-    
+
+    def setActivityMonitor(self, am):
+        self._activity_monitor = am
+
     def setVersionCacheDeactivateAfter(self, v):
         self._version_cache_deactivate_after=v
         for ver in self._pools[0].keys():