[Zodb-checkins] CVS: ZODB3/BDBStorage - BDBFullStorage.py:1.44.4.3
Barry Warsaw
barry@wooz.org
Tue, 21 Jan 2003 17:28:41 -0500
Update of /cvs-repository/ZODB3/BDBStorage
In directory cvs.zope.org:/tmp/cvs-serv26584
Modified Files:
Tag: ZODB3-3_1-branch
BDBFullStorage.py
Log Message:
Backporting of various changes from the 3.2 branch. Specifically:
- generalize the table for storage metadata, i.e. packtime -> info
- get ZERO from the package
- close the pack race condition
=== ZODB3/BDBStorage/BDBFullStorage.py 1.44.4.2 => 1.44.4.3 ===
--- ZODB3/BDBStorage/BDBFullStorage.py:1.44.4.2 Tue Jan 7 14:38:52 2003
+++ ZODB3/BDBStorage/BDBFullStorage.py Tue Jan 21 17:28:36 2003
@@ -1,6 +1,6 @@
##############################################################################
#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# Copyright (c) 2001 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
@@ -13,9 +13,9 @@
##############################################################################
"""Berkeley storage with full undo and versioning support.
-"""
-__version__ = '$Revision$'.split()[-2:][0]
+$Revision$
+"""
import time
import cPickle as pickle
@@ -27,13 +27,12 @@
from ZODB.TimeStamp import TimeStamp
from ZODB.ConflictResolution import ConflictResolvingStorage, ResolvedSerial
-from BDBStorage import db
+from BDBStorage import db, ZERO
from BDBStorage.BerkeleyBase import BerkeleyBase, PackStop, _WorkThread
ABORT = 'A'
COMMIT = 'C'
PRESENT = 'X'
-ZERO = '\0'*8
# Special flag for uncreated objects (i.e. Does Not Exist)
DNE = '\377'*8
@@ -178,9 +177,15 @@
# pending table is empty, the oids, pvids, and prevrevids tables
# must also be empty.
#
- # packtime -- tid
- # The time of the last pack. It is illegal to undo to before the
- # last pack time.
+ # info -- {key -> value}
+ # This table contains storage metadata information. The keys and
+ # values are simple strings of variable length. Here are the
+ # valid keys:
+ #
+ # packtime - time of the last pack. It is illegal to undo to
+ # before the last pack time.
+ #
+ # version - the version of the database (reserved for ZODB4)
#
# objrevs -- {newserial+oid -> oldserial}
# This table collects object revision information for packing
@@ -204,6 +209,8 @@
# This table is a Queue, not a BTree. It is used during the mark
# phase of pack() and contains a list of oids for work to be done.
#
+ self._packing = False
+ self._info = self._setupDB('info')
self._serials = self._setupDB('serials', db.DB_DUP)
self._pickles = self._setupDB('pickles')
self._refcounts = self._setupDB('refcounts')
@@ -223,7 +230,6 @@
# Tables to support packing.
self._objrevs = self._setupDB('objrevs', db.DB_DUP)
self._packmark = self._setupDB('packmark')
- self._packtime = self._setupDB('packtime')
self._oidqueue = self._setupDB('oidqueue', 0, db.DB_QUEUE, 8)
self._delqueue = self._setupDB('delqueue', 0, db.DB_QUEUE, 8)
# Do recovery and consistency checks
@@ -429,10 +435,18 @@
refcount = self._refcounts.get(oid, ZERO, txn=txn)
self._refcounts.put(oid, incr(refcount, delta), txn=txn)
# Now clean up the temporary log tables
- self._oids.truncate(txn)
self._pvids.truncate(txn)
self._prevrevids.truncate(txn)
self._pending.truncate(txn)
+ # If we're in the middle of a pack, we need to add to the packmark
+ # table any objects that were modified in this transaction.
+ # Otherwise, there's a race condition where mark might have happened,
+ # then the object is added, then sweep runs, deleting the object
+ # created in the interrim.
+ if self._packing:
+ for oid in self._oids.keys():
+ self._packmark.put(oid, PRESENT, txn=txn)
+ self._oids.truncate(txn)
def _dobegin(self, txn, tid, u, d, e):
# When a transaction begins, we set the pending flag to ABORT,
@@ -1032,13 +1046,7 @@
self._lock_release()
def _last_packtime(self):
- packtimes = self._packtime.keys()
- if len(packtimes) == 1:
- return packtimes[0]
- elif len(packtimes) == 0:
- return ZERO
- else:
- assert False, 'too many packtimes'
+ return self._info.get('packtime', ZERO)
def lastTransaction(self):
"""Return transaction id for last committed transaction"""
@@ -1306,11 +1314,10 @@
finally:
self._lock_release()
- #
# Packing
#
# There are two types of pack operations, the classic pack and the
- # autopack. Autopack's sole job is to periodically delete non-current
+ # autopack. Autopack's primary job is to periodically delete non-current
# object revisions. It runs in a thread and has an `autopack time' which
# is essentially just a time in the past at which to autopack to. For
# example, you might set up autopack to run once per hour, packing away
@@ -1333,7 +1340,6 @@
# acquisition as granularly as possible so that packing doesn't block
# other operations for too long. But remember we don't use Berkeley locks
# so we have to be careful about our application level locks.
- #
# First, the public API for classic pack
def pack(self, t, zreferencesf):
@@ -1346,6 +1352,7 @@
# A simple wrapper around the bulk of packing, but which acquires a
# lock that prevents multiple packs from running at the same time.
self._packlock.acquire()
+ self._packing = True
try:
# We don't wrap this in _withtxn() because we're going to do the
# operation across several Berkeley transactions, which allows
@@ -1353,6 +1360,7 @@
# done.
self._dopack(t)
finally:
+ self._packing = False
self._packlock.release()
self.log('classic pack finished')
@@ -1417,6 +1425,7 @@
# A simple wrapper around the bulk of packing, but which acquires a
# lock that prevents multiple packs from running at the same time.
self._packlock.acquire()
+ self._packing = True
try:
# We don't wrap this in _withtxn() because we're going to do the
# operation across several Berkeley transactions, which allows
@@ -1424,6 +1433,7 @@
# done.
self._dopack(t, gc)
finally:
+ self._packing = False
self._packlock.release()
self.log('autopack finished')
@@ -1471,10 +1481,9 @@
if co: co.close()
if ct: ct.close()
# Note that before we commit this Berkeley transaction, we also need
- # to update the packtime table, so we can't have the possibility of a
- # race condition with undoLog().
- self._packtime.truncate(txn)
- self._packtime.put(packtid, PRESENT, txn=txn)
+ # to update the last packtime entry, so we can't have the possibility
+ # of a race condition with undoLog().
+ self._info.put('packtime', packtid, txn=txn)
def _decrefPickle(self, oid, lrevid, txn):
if lrevid == DNE:
@@ -1601,7 +1610,6 @@
# oidqueue is a BerkeleyDB Queue that holds the list of object ids to
# look at next, and by using this we don't need to keep an in-memory
# dictionary.
- assert len(self._packmark) == 0
assert len(self._oidqueue) == 0
# Quick exit for empty storages
if not self._serials: