[Checkins] SVN: zc.zodbdgc/trunk/src/zc/zodbdgc/ Deleting 10000 objects at a time was found to put too much stress on a
Jim Fulton
jim at zope.com
Tue Sep 8 06:31:40 EDT 2009
Log message for revision 103637:
Deleting 10000 objects at a time was found to put too much stress on a
heavily loaded storage server.
- Add a sleep or allow the storahe to rest after a set of deletions.
Sleep for twice the time taken to perform the deletions.
- Adjust the deletion batch size to take about .5 seconds per
batch of deletions, but do at least 10 at a time.
Changed:
U zc.zodbdgc/trunk/src/zc/zodbdgc/README.test
U zc.zodbdgc/trunk/src/zc/zodbdgc/__init__.py
-=-
Modified: zc.zodbdgc/trunk/src/zc/zodbdgc/README.test
===================================================================
--- zc.zodbdgc/trunk/src/zc/zodbdgc/README.test 2009-09-08 10:11:10 UTC (rev 103636)
+++ zc.zodbdgc/trunk/src/zc/zodbdgc/README.test 2009-09-08 10:31:39 UTC (rev 103637)
@@ -192,7 +192,7 @@
Now let's perform gc.
>>> import zc.zodbdgc
- >>> bad = zc.zodbdgc.gc('config', days=2, batch_size=2)
+ >>> bad = zc.zodbdgc.gc('config', days=2)
>>> for name, oid in sorted(bad.iterator()):
... print name, u64(oid)
Modified: zc.zodbdgc/trunk/src/zc/zodbdgc/__init__.py
===================================================================
--- zc.zodbdgc/trunk/src/zc/zodbdgc/__init__.py 2009-09-08 10:11:10 UTC (rev 103636)
+++ zc.zodbdgc/trunk/src/zc/zodbdgc/__init__.py 2009-09-08 10:31:39 UTC (rev 103637)
@@ -89,16 +89,16 @@
fs=dict(o.split('=') for o in options.fs or ()))
-def gc(conf, days=1, ignore=(), conf2=None, batch_size=10000, fs=()):
+def gc(conf, days=1, ignore=(), conf2=None, fs=()):
close = []
try:
- return gc_(close, conf, days, ignore, conf2, batch_size, fs)
+ return gc_(close, conf, days, ignore, conf2, fs)
finally:
for db in close:
for db in db.databases.itervalues():
db.close()
-def gc_(close, conf, days, ignore, conf2, batch_size, fs):
+def gc_(close, conf, days, ignore, conf2, fs):
db1 = ZODB.config.databaseFromFile(open(conf))
close.append(db1)
if conf2 is None:
@@ -207,12 +207,14 @@
close.pop()
# Now, we have the garbage in bad. Remove it.
+ batch_size = 100
for name, db in sorted(db1.databases.iteritems()):
logger.info("%s: remove garbage", name)
storage = db.storage
t = transaction.begin()
storage.tpc_begin(t)
nd = 0
+ start = time.time()
for oid, tid in bad.iterator(name):
try:
storage.deleteObject(oid, tid, t)
@@ -225,6 +227,9 @@
storage.tpc_finish(t)
t.commit()
logger.info("%s: deleted %s", name, nd)
+ duration = time.time()-start
+ time.sleep(duration*2)
+ batch_size = max(10, int(batch_size*.5/duration))
t = transaction.begin()
storage.tpc_begin(t)
More information about the checkins
mailing list