[Zodb-checkins] SVN: ZODB/branches/perfmetrics/s Send pool sizes, load/store counts, and connection durations to Statsd.
Shane Hathaway
cvs-admin at zope.org
Mon Sep 3 20:35:59 UTC 2012
Log message for revision 127681:
Send pool sizes, load/store counts, and connection durations to Statsd.
Changed:
U ZODB/branches/perfmetrics/setup.py
U ZODB/branches/perfmetrics/src/ZODB/DB.py
U ZODB/branches/perfmetrics/src/ZODB/tests/testDB.py
-=-
Modified: ZODB/branches/perfmetrics/setup.py
===================================================================
--- ZODB/branches/perfmetrics/setup.py 2012-09-03 18:12:38 UTC (rev 127680)
+++ ZODB/branches/perfmetrics/setup.py 2012-09-03 20:35:55 UTC (rev 127681)
@@ -20,7 +20,7 @@
interface, rich transaction support, and undo.
"""
-VERSION = "3.10dev"
+VERSION = "3.10.5-perfmetrics"
from ez_setup import use_setuptools
use_setuptools()
@@ -200,6 +200,7 @@
tests_require = ['zope.testing', manuel_version],
extras_require = dict(test=['zope.testing', manuel_version]),
install_requires = [
+ 'perfmetrics',
transaction_version,
'zc.lockfile',
'ZConfig',
Modified: ZODB/branches/perfmetrics/src/ZODB/DB.py
===================================================================
--- ZODB/branches/perfmetrics/src/ZODB/DB.py 2012-09-03 18:12:38 UTC (rev 127680)
+++ ZODB/branches/perfmetrics/src/ZODB/DB.py 2012-09-03 20:35:55 UTC (rev 127681)
@@ -35,6 +35,7 @@
from ZODB.interfaces import IMVCCStorage
import transaction
+from perfmetrics import statsd_client
from persistent.TimeStamp import TimeStamp
@@ -319,6 +320,20 @@
return before
+class StatsdActivityMonitor(object):
+ # Implements the ActivityMonitor interface.
+
+ def closedConnection(self, connection):
+ client = statsd_client()
+ if client is not None:
+ database_name = connection.db().database_name or '_'
+ loads, stores = connection.getTransferCounts(True)
+ buf = []
+ client.incr('zodb.%s.loads' % database_name, loads, buf=buf)
+ client.incr('zodb.%s.stores' % database_name, stores, buf=buf)
+ client.sendbuf(buf)
+
+
class DB(object):
"""The Object Database
-------------------
@@ -362,7 +377,8 @@
implements(IDatabase)
klass = Connection # Class to use for connections
- _activity_monitor = next = previous = None
+ _activity_monitor = StatsdActivityMonitor()
+ next = previous = None
def __init__(self, storage,
pool_size=7,
@@ -489,12 +505,23 @@
self._a()
try:
assert connection._db is self
+
+ client = statsd_client()
+ if client is not None:
+ if connection.opened:
+ elapsed = int((time.time() - connection.opened) * 1000.0)
+ dbname = self.database_name or '_'
+ client.timing('zodb.%s.duration' % dbname, elapsed)
+
connection.opened = None
if connection.before:
self.historical_pool.repush(connection, connection.before)
else:
self.pool.repush(connection)
+
+ if client is not None:
+ self._reportPoolInfo()
finally:
self._r()
@@ -755,6 +782,9 @@
self.pool.availableGC()
self.historical_pool.availableGC()
+ if statsd_client() is not None:
+ self._reportPoolInfo()
+
return result
finally:
@@ -790,6 +820,38 @@
def getActivityMonitor(self):
return self._activity_monitor
+ def _reportPoolInfo(self):
+ """Report pool stats to statsd.
+
+ The lock must be acquired when this is called.
+ """
+ client = statsd_client()
+ if client is None:
+ return
+
+ buf = []
+ dbname = self.database_name or '_'
+
+ for pool, prefix in [(self.pool, ''),
+ (self.historical_pool, 'hist_')]:
+ name = 'zodb.%s.%s' % (dbname, prefix)
+ conns = []
+ pool.all.map(conns.append)
+ opened = 0
+ estsize = 0
+ nonghosts = 0
+ for c in conns:
+ if c.opened:
+ opened += 1
+ estsize += c._cache.total_estimated_size
+ nonghosts += c._cache.cache_non_ghost_count
+ client.gauge(name + 'pool', len(conns), buf=buf)
+ client.gauge(name + 'opened', opened, buf=buf)
+ client.gauge(name + 'estsize', estsize, buf=buf)
+ client.gauge(name + 'nonghosts', nonghosts, buf=buf)
+
+ client.sendbuf(buf)
+
def pack(self, t=None, days=0):
"""Pack the storage, deleting unused object revisions.
Modified: ZODB/branches/perfmetrics/src/ZODB/tests/testDB.py
===================================================================
--- ZODB/branches/perfmetrics/src/ZODB/tests/testDB.py 2012-09-03 18:12:38 UTC (rev 127680)
+++ ZODB/branches/perfmetrics/src/ZODB/tests/testDB.py 2012-09-03 20:35:55 UTC (rev 127681)
@@ -64,7 +64,64 @@
import ZODB.serialize
self.assert_(self.db.references is ZODB.serialize.referencesf)
+ def test_open_and_close_with_statsd_client(self):
+ from perfmetrics import statsd_client_stack
+ sent = []
+ class DummyStatsdClient:
+ def gauge(self, name, value, buf):
+ buf.append('%s:%s|g' % (name, value))
+
+ def incr(self, name, value, buf):
+ buf.append('%s:%s|c' % (name, value))
+
+ def timing(self, name, value):
+ sent.append('%s:%s|ms' % (name, value))
+
+ def sendbuf(self, buf):
+ sent.append(buf)
+
+ client = DummyStatsdClient()
+ statsd_client_stack.push(client)
+ try:
+ c = self.db.open()
+ c.root().get('x')
+ time.sleep(0.1)
+ c.close()
+ finally:
+ statsd_client_stack.pop()
+
+ self.maxDiff = 10000
+ self.assertEqual(len(sent), 4)
+ self.assertEqual(sent[0],
+ ['zodb.unnamed.pool:1|g',
+ 'zodb.unnamed.opened:1|g',
+ 'zodb.unnamed.estsize:0|g',
+ 'zodb.unnamed.nonghosts:0|g',
+ 'zodb.unnamed.hist_pool:0|g',
+ 'zodb.unnamed.hist_opened:0|g',
+ 'zodb.unnamed.hist_estsize:0|g',
+ 'zodb.unnamed.hist_nonghosts:0|g'])
+
+ import re
+ mo = re.match(r'zodb.unnamed.duration:([0-9\.]+)|ms', sent[1])
+ self.assertTrue(mo is not None, 'Regex failed to match %r' % sent[1])
+ self.assertTrue(int(mo.group(1)) > 0)
+
+ self.assertEqual(sent[2],
+ ['zodb.unnamed.pool:1|g',
+ 'zodb.unnamed.opened:0|g',
+ 'zodb.unnamed.estsize:64|g',
+ 'zodb.unnamed.nonghosts:1|g',
+ 'zodb.unnamed.hist_pool:0|g',
+ 'zodb.unnamed.hist_opened:0|g',
+ 'zodb.unnamed.hist_estsize:0|g',
+ 'zodb.unnamed.hist_nonghosts:0|g'])
+ self.assertEqual(sent[3],
+ ['zodb.unnamed.loads:1|c',
+ 'zodb.unnamed.stores:0|c'])
+
+
def test_invalidateCache():
"""The invalidateCache method invalidates a connection caches for all of
the connections attached to a database::
More information about the Zodb-checkins
mailing list