[Zodb-checkins] CVS: ZODB3/bsddb3Storage/bsddb3Storage - Autopack.py:1.5 BerkeleyBase.py:1.20 Full.py:1.46 Minimal.py:1.13 CommitLog.py:NONE
Barry Warsaw
barry@wooz.org
Tue, 5 Nov 2002 18:07:33 -0500
Update of /cvs-repository/ZODB3/bsddb3Storage/bsddb3Storage
In directory cvs.zope.org:/tmp/cvs-serv1143/bsddb3Storage/bsddb3Storage
Modified Files:
Autopack.py BerkeleyBase.py Full.py Minimal.py
Removed Files:
CommitLog.py
Log Message:
Merging the Berkeley storage's bdb-nolocks branch back into the trunk
for ZODB 3.2.
=== ZODB3/bsddb3Storage/bsddb3Storage/Autopack.py 1.4 => 1.5 ===
--- ZODB3/bsddb3Storage/bsddb3Storage/Autopack.py:1.4 Fri Jul 19 12:42:37 2002
+++ ZODB3/bsddb3Storage/bsddb3Storage/Autopack.py Tue Nov 5 18:07:31 2002
@@ -21,7 +21,7 @@
import struct
import time
-# This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
+# This uses the Dunn/Kuchling PyBSDDB3 extension module available from
# http://pybsddb.sourceforge.net
from bsddb3 import db
@@ -61,7 +61,7 @@
# base class infrastructure and are shared by the Minimal
# implementation.
#
- # serials -- {oid -> serial}
+ # serials -- {oid+tid -> serial}
# Maps oids to object serial numbers. The serial number is
# essentially a timestamp used to determine if conflicts have
# arisen, and serial numbers double as transaction ids and object
@@ -104,6 +104,32 @@
self._oids.close()
BerkeleyBase.close(self)
+ def _getSerial(self, oid):
+ c = self._serials.cursor()
+ try:
+ lastvalue = None
+ # Search for the largest oid+revid key in the serials table that
+ # doesn't have a revid component equal to the current revid.
+ try:
+ rec = c.set_range(oid)
+ except db.DBNotFoundError:
+ rec = None
+ while rec:
+ key, value = rec
+ koid = key[:8]
+ ktid = key[8:]
+ if koid <> oid:
+ break
+ lastvalue = value
+ if ktid == self._serial:
+ break
+ rec = c.next()
+ if lastvalue is None:
+ return None
+ return lastvalue[:8]
+ finally:
+ c.close()
+
def _begin(self, tid, u, d, e):
# Nothing needs to be done
pass
@@ -112,12 +138,41 @@
# Nothing needs to be done, but override the base class's method
pass
+ def store(self, oid, serial, data, version, transaction):
+ self._lock_acquire()
+ try:
+ # Transaction guard
+ if transaction is not self._transaction:
+ raise POSException.StorageTransactionError(self, transaction)
+ # We don't support versions
+ if version <> '':
+ raise POSException.Unsupported, 'versions are not supported'
+ oserial = self._getSerial(oid)
+ if oserial is not None and serial <> oserial:
+ # BAW: Here's where we'd try to do conflict resolution
+ raise POSException.ConflictError(serials=(oserial, serial))
+ tid = self._serial
+ txn = self._env.txn_begin()
+ try:
+ self._serials.put(oid+tid, self._serial, txn=txn)
+ self._pickles.put(oid+tid, data, txn=txn)
+ self._actions.put(tid+oid, INC, txn=txn)
+ self._oids.put(oid, ' ', txn=txn)
+ except:
+ txn.abort()
+ raise
+ else:
+ txn.commit()
+ return self._serial
+ finally:
+ self._lock_release()
+
def _finish(self, tid, u, d, e):
# TBD: what about u, d, and e?
#
# First, append a DEL to the actions for each old object, then update
# the current serials table so that its revision id points to this
- # trancation id.
+ # transaction id.
txn = self._env.txn_begin()
try:
c = self._oids.cursor()
@@ -128,8 +183,8 @@
lastrevid = self._serials.get(oid, txn=txn)
if lastrevid:
self._actions.put(lastrevid+oid, DEC, txn=txn)
- self._serials.put(oid, tid, txn=txn)
rec = c.next()
+ self._oids.truncate()
finally:
c.close()
except:
@@ -137,7 +192,6 @@
raise
else:
txn.commit()
- self._oids.truncate()
# Override BerkeleyBase._abort()
def _abort(self):
@@ -164,30 +218,6 @@
self._oids.truncate()
self._transaction.abort()
- def store(self, oid, serial, data, version, transaction):
- # Transaction guard
- if transaction is not self._transaction:
- raise POSException.StorageTransactionError(self, transaction)
- # We don't support versions
- if version <> '':
- raise POSException.Unsupported, 'versions are not supported'
- oserial = self._serials.get(oid)
- if oserial is not None and serial <> oserial:
- # BAW: Here's where we'd try to do conflict resolution
- raise POSException.ConflictError(serials=(oserial, serial))
- tid = self._serial
- txn = self._env.txn_begin()
- try:
- self._pickles.put(oid+tid, data, txn=txn)
- self._actions.put(tid+oid, INC, txn=txn)
- self._oids.put(oid, ' ', txn=txn)
- except:
- txn.abort()
- raise
- else:
- txn.commit()
- return self._serial
-
def load(self, oid, version):
if version <> '':
raise POSException.Unsupported, 'versions are not supported'
@@ -196,6 +226,7 @@
def loadSerial(self, oid, serial):
current = self._serials[oid]
+ # BAW: should we allow older serials to be retrieved?
if current == serial:
return self._pickles[oid+current]
else:
=== ZODB3/bsddb3Storage/bsddb3Storage/BerkeleyBase.py 1.19 => 1.20 ===
--- ZODB3/bsddb3Storage/bsddb3Storage/BerkeleyBase.py:1.19 Tue Sep 3 14:07:30 2002
+++ ZODB3/bsddb3Storage/bsddb3Storage/BerkeleyBase.py Tue Nov 5 18:07:31 2002
@@ -2,14 +2,14 @@
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
-#
+#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
-#
+#
##############################################################################
"""Base class for BerkeleyStorage implementations.
@@ -23,84 +23,69 @@
# http://pybsddb.sourceforge.net
from bsddb3 import db
-# BaseStorage provides some common storage functionality. It is derived from
-# UndoLogCompatible.UndoLogCompatible, which "[provides] backward
-# compatability with storages that have undoLog, but not undoInfo."
-#
-# BAW: I'm not entirely sure what that means, but the UndoLogCompatible
-# subclass provides one method:
-#
-# undoInfo(first, last, specification). Unfortunately this method appears to
-# be undocumented. Jeremy tells me it's still required though.
-#
-# BaseStorage provides primitives for lock acquisition and release,
-# abortVersion(), commitVersion() and a host of other methods, some of which
-# are overridden here, some of which are not.
+# BaseStorage provides primitives for lock acquisition and release, and a host
+# of other methods, some of which are overridden here, some of which are not.
from ZODB import POSException
+from ZODB.lock_file import lock_file
from ZODB.BaseStorage import BaseStorage
+from ZODB.referencesf import referencesf
-__version__ = '$Revision$'.split()[-2:][0]
+GBYTES = 1024 * 1024 * 1000
-# Lock usage is inherently unbounded because there may be an unlimited number
-# of objects actually touched in any single transaction, and worst case could
-# be that each object is on a different page in the database. Berkeley BTrees
-# implement a lock per leaf page, plus a lock per level. We try to limit the
-# negative effects of this by writing as much data optimistically as we can.
-# But there's no way to completely avoid this. So this value is used to size
-# the lock subsystem before the environment is opened.
-DEFAULT_MAX_LOCKS = 20000
+__version__ = '$Revision$'.split()[-2:][0]
+
class BerkeleyConfig:
"""Bag of bits for describing various underlying configuration options.
Berkeley databases are wildly configurable, and this class exposes some of
- that. Two important configuration options are the size of the lock table
- and the checkpointing policy. To customize these options, instantiate one
- of these classes and set the attributes below to the desired value. Then
- pass this instance to the Berkeley storage constructor, using the `config'
- keyword argument.
-
- Locks in Berkeley are a limited and static resource; they can only be
- changed before the environment is opened. It is possible for Berkeley
- based storages to exhaust the available locks because worst case is to
- consume one lock per object being modified, and transactions are unbounded
- in the number of objects they modify. See
-
- http://www.sleepycat.com/docs/ref/lock/max.html
-
- for a discussion on lock sizing. These attributes control the lock
- sizing:
-
- - numlocks is passed directly to set_lk_max_locks() when the environment
- is opened.
-
- You will need to find the right balance between the number of locks
- allocated and the system resources that consumes. If the locks are
- exhausted a TransactionTooLargeError can get raised during commit.
-
- To improve recovery times in case of failures, you should set up a
- checkpointing policy when you create the database. Note that the database
- is automatically, and forcefully, checkpointed twice when it is closed.
- But an exception during processing (e.g.
+ that. To customize these options, instantiate one of these classes and
+ set the attributes below to the desired value. Then pass this instance to
+ the Berkeley storage constructor, using the `config' keyword argument.
+
+ Berkeley storages need to be checkpointed occasionally, otherwise
+ automatic recover can take a huge amount of time. You should set up a
+ checkpointing policy which trades off the amount of work done periodically
+ against the recovery time. Note that the Berkeley environment is
+ automatically, and forcefully, checkpointed twice when it is closed.
The following checkpointing attributes are supported:
- - interval indicates the maximum number of calls to tpc_finish() after
- which a checkpoint is performed.
+ - interval indicates the approximate number of Berkeley transaction
+ commits and aborts after which a checkpoint is performed. Berkeley
+ transactions are performed after ZODB aborts, commits, and stores.
- kbytes is passed directly to txn_checkpoint()
- min is passed directly to txn_checkpoint()
+ You can acheive one of the biggest performance wins by moving the Berkeley
+ log files to a different disk than the data files. We saw between 2.5 and
+ 7 x better performance this way. Here are attributes which control the
+ log files.
+
- logdir if not None, is passed to the environment's set_lg_dir() method
before it is opened.
+
+ You can also improve performance by tweaking the Berkeley cache size.
+ Berkeley's default cache size is 256KB which is usually too small. Our
+ default cache size is 128MB which seems like a useful tradeoff between
+ resource consumption and improved performance. You might be able to get
+ slightly better results by turning up the cache size, although be mindful
+ of your system's limits. See here for more details:
+
+ http://www.sleepycat.com/docs/ref/am_conf/cachesize.html
+
+ These attributes control cache size settings:
+
+ - cachesize should be the size of the cache in bytes.
"""
- numlocks = DEFAULT_MAX_LOCKS
interval = 100
kbyte = 0
min = 0
logdir = None
+ cachesize = 128 * 1024 * 1024
@@ -153,7 +138,7 @@
if env == '':
raise TypeError, 'environment name is empty'
elif isinstance(env, StringType):
- self._env = env_from_string(env, self._config)
+ self._env, self._lockfile = env_from_string(env, self._config)
else:
self._env = env
@@ -161,22 +146,12 @@
# Initialize a few other things
self._prefix = prefix
- self._commitlog = None
# Give the subclasses a chance to interpose into the database setup
# procedure
self._setupDBs()
# Initialize the object id counter.
self._init_oid()
- def _closelog(self):
- if self._commitlog:
- self._commitlog.finish()
- # JF: unlinking might be too inefficient. JH: might use mmap
- # files. BAW: maybe just truncate the file, or write a length
- # into the headers and just zero out the length.
- self._commitlog.close(unlink=1)
- self._commitlog = None
-
def _setupDB(self, name, flags=0):
"""Open an individual database with the given flags.
@@ -229,7 +204,7 @@
# BAW: the last parameter is undocumented in the UML model
if self._len is not None:
# Increment the cached length
- self._len = self._len + 1
+ self._len += 1
return BaseStorage.new_oid(self, last)
def getSize(self):
@@ -238,22 +213,8 @@
filename = os.path.join(self._env.db_home, 'zodb_pickles')
return os.path.getsize(filename)
- # BAW: this overrides BaseStorage.tpc_vote() with exactly the same
- # implementation. This is so Zope 2.3.1, which doesn't include the change
- # to BaseStorage, will work with Berkeley. Once we can ignore older
- # versions of ZODB, we can get rid of this.
- def tpc_vote(self, transaction):
- self._lock_acquire()
- try:
- if transaction is not self._transaction: return
- self._vote()
- finally:
- self._lock_release()
-
def _vote(self):
- # Make a promise to commit all the registered changes. Rewind and put
- # our commit log in the PROMISED state.
- self._commitlog.promise()
+ pass
def _finish(self, tid, user, desc, ext):
"""Called from BaseStorage.tpc_finish(), this commits the underlying
@@ -272,7 +233,6 @@
"""Called from BaseStorage.tpc_abort(), this aborts the underlying
BSDDB transaction.
"""
- self._closelog()
self._transaction.abort()
def _clear_temp(self):
@@ -295,24 +255,40 @@
# was shutdown gracefully. The DB_FORCE flag is required for
# the second checkpoint, but we include it in both because it
# can't hurt and is more robust.
- self._env.txn_checkpoint(0, 0, db.DB_FORCE)
- self._env.txn_checkpoint(0, 0, db.DB_FORCE)
+ self._env.txn_checkpoint(0, 0, db.DB_FORCE)
+ self._env.txn_checkpoint(0, 0, db.DB_FORCE)
+ lockfile = os.path.join(self._env.db_home, '.lock')
+ self._lockfile.close()
self._env.close()
- self._closelog()
-
- # Useful for debugging
-
- def _lockstats(self):
- d = self._env.lock_stat()
- return 'locks = [%(nlocks)d/%(maxnlocks)d]' % d
+ os.unlink(lockfile)
def _docheckpoint(self):
+ # Periodically checkpoint the database. This is called approximately
+ # once per Berkeley transaction commit or abort.
config = self._config
config._counter += 1
if config._counter > config.interval:
self._env.txn_checkpoint(config.kbyte, config.min)
config._counter = 0
+ def _update(self, deltas, data, incdec):
+ refdoids = []
+ referencesf(data, refdoids)
+ for oid in refdoids:
+ rc = deltas.get(oid, 0) + incdec
+ if rc == 0:
+ # Save space in the dict by zapping zeroes
+ del deltas[oid]
+ else:
+ deltas[oid] = rc
+
+ def _withlock(self, meth, *args):
+ self._lock_acquire()
+ try:
+ return meth(*args)
+ finally:
+ self._lock_release()
+
def env_from_string(envname, config):
@@ -323,16 +299,30 @@
except OSError, e:
if e.errno <> errno.EEXIST: raise
# already exists
+ # Create the lock file so no other process can open the environment.
+ # This is required in order to work around the Berkeley lock
+ # exhaustion problem (i.e. we do our own application level locks
+ # rather than rely on Berkeley's finite page locks).
+ lockpath = os.path.join(envname, '.lock')
+ try:
+ lockfile = open(lockpath, 'r+')
+ except IOError, e:
+ if e.errno <> errno.ENOENT: raise
+ lockfile = open(lockpath, 'w+')
+ lock_file(lockfile)
+ lockfile.write(str(os.getpid()))
+ lockfile.flush()
+ # Create, initialize, and open the environment
env = db.DBEnv()
- env.set_lk_max_locks(config.numlocks)
if config.logdir is not None:
env.set_lg_dir(config.logdir)
+ gbytes, bytes = divmod(config.cachesize, GBYTES)
+ env.set_cachesize(gbytes, bytes)
env.open(envname,
db.DB_CREATE # create underlying files as necessary
| db.DB_RECOVER # run normal recovery before opening
| db.DB_INIT_MPOOL # initialize shared memory buffer pool
- | db.DB_INIT_LOCK # initialize locking subsystem
| db.DB_INIT_TXN # initialize transaction subsystem
| db.DB_THREAD # we use the environment from other threads
)
- return env
+ return env, lockfile
=== ZODB3/bsddb3Storage/bsddb3Storage/Full.py 1.45 => 1.46 === (2440/2540 lines abridged)
--- ZODB3/bsddb3Storage/bsddb3Storage/Full.py:1.45 Fri Oct 18 16:33:51 2002
+++ ZODB3/bsddb3Storage/bsddb3Storage/Full.py Tue Nov 5 18:07:31 2002
@@ -13,25 +13,18 @@
##############################################################################
"""Berkeley storage with full undo and versioning support.
-
-See Minimal.py for an implementation of Berkeley storage that does not support
-undo or versioning.
"""
__version__ = '$Revision$'.split()[-2:][0]
import sys
-import struct
import time
-
-from cPickle import loads, Pickler
-Pickler = Pickler()
-Pickler.fast = 1 # Don't use a memo
-fast_pickle_dumps = Pickler.dump
-del Pickler
+import cPickle as pickle
+from struct import pack, unpack
# This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
-# http://pybsddb.sourceforge.net
+# http://pybsddb.sourceforge.net. It is compatible with release 3.4 of
+# PyBSDDB3.
from bsddb3 import db
from ZODB import POSException
@@ -39,6 +32,7 @@
from ZODB.referencesf import referencesf
from ZODB.TimeStamp import TimeStamp
from ZODB.ConflictResolution import ConflictResolvingStorage, ResolvedSerial
+import zLOG
import ThreadLock
# BerkeleyBase.BerkeleyBase class provides some common functionality for both
@@ -46,7 +40,6 @@
# ZODB.BaseStorage.BaseStorage which itself provides some common storage
# functionality.
from BerkeleyBase import BerkeleyBase
-from CommitLog import FullLog
# Flags for transaction status in the transaction metadata table. You can
# only undo back to the last pack, and any transactions before the pack time
@@ -56,6 +49,9 @@
UNDOABLE_TRANSACTION = 'Y'
[-=- -=- -=- 2440 lines omitted -=- -=- -=-]
"""
- def __init__(self, storage):
+ def __init__(self, storage, start, stop):
self._storage = storage
- self._tid = None
- self._closed = 0
+ self._tid = start
+ self._stop = stop
+ self._closed = False
+ self._first = True
- def __getitem__(self, i):
+ def next(self):
"""Return the ith item in the sequence of transaction data.
Items must be accessed sequentially, and are instances of
@@ -1532,16 +1678,21 @@
if self._closed:
raise IOError, 'iterator is closed'
# Let IndexErrors percolate up.
- tid, status, user, desc, ext = self._storage._nexttxn(self._tid)
+ tid, status, user, desc, ext = self._storage._nexttxn(
+ self._tid, self._first)
+ self._first = False
+ # Did we reach the specified end?
+ if self._stop is not None and tid > self._stop:
+ raise IndexError
self._tid = tid
return _RecordsIterator(self._storage, tid, status, user, desc, ext)
def close(self):
- self._closed = 1
+ self._closed = True
-class _RecordsIterator:
+class _RecordsIterator(_GetItemBase):
"""Provide transaction meta-data and forward iteration through the
transactions in a storage.
@@ -1580,7 +1731,7 @@
# To make .pop() more efficient
self._oids.reverse()
- def __getitem__(self, i):
+ def next(self):
"""Return the ith item in the sequence of record data.
Items must be accessed sequentially, and are instances of Record. An
=== ZODB3/bsddb3Storage/bsddb3Storage/Minimal.py 1.12 => 1.13 === (417/517 lines abridged)
--- ZODB3/bsddb3Storage/bsddb3Storage/Minimal.py:1.12 Tue Feb 12 17:33:09 2002
+++ ZODB3/bsddb3Storage/bsddb3Storage/Minimal.py Tue Nov 5 18:07:32 2002
@@ -2,247 +2,327 @@
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
-#
+#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
-#
+#
##############################################################################
"""Berkeley storage without undo or versioning.
-
-See Full.py for an implementation of Berkeley storage that does support undo
-and versioning.
"""
__version__ = '$Revision$'[-2:][0]
# This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
-# http://pybsddb.sourceforge.net. It is compatible with release 3.0 of
+# http://pybsddb.sourceforge.net. It is compatible with release 3.4 of
# PyBSDDB3.
from bsddb3 import db
-# BerkeleyBase.BerkeleyBase class provides some common functionality for both
-# the Full and Minimal implementations. It in turn inherits from
-# ZODB.BaseStorage.BaseStorage which itself provides some common storage
-# functionality.
+# BerkeleyBase class provides some common functionality for BerkeleyDB-based
+# storages. It in turn inherits from BaseStorage which itself provides some
+# common storage functionality.
from BerkeleyBase import BerkeleyBase
-from CommitLog import PacklessLog
from ZODB import POSException
-from ZODB import utils
+from ZODB.utils import U64, p64
+from ZODB.referencesf import referencesf
+
+ABORT = 'A'
+COMMIT = 'C'
+PRESENT = 'X'
+ZERO = '\0'*8
[-=- -=- -=- 417 lines omitted -=- -=- -=-]
- for oid in self._pickles.keys():
- if not seen.has_key(oid):
- del self._pickles[oid]
+ # If there is exactly one entry then this has to be the entry for
+ # the object, regardless of the pending flag.
+ #
+ # If there are two entries, then we need to look at the pending
+ # flag to decide which to return (there /better/ be a pending flag
+ # set!). If the pending flag is COMMIT then we've already voted
+ # so the second one is the good one. If the pending flag is ABORT
+ # then we haven't yet committed to this transaction so the first
+ # one is the good one.
+ serials = []
+ try:
+ rec = c.set(oid)
+ except db.DBNotFoundError:
+ rec = None
+ while rec:
+ serials.append(rec[1])
+ rec = c.next_dup()
+ if not serials:
+ return None
+ if len(serials) == 1:
+ return serials[0]
+ pending = self._pending.get(self._serial)
+ assert pending in (ABORT, COMMIT)
+ if pending == ABORT:
+ return serials[0]
+ return serials[1]
+ finally:
+ c.close()
+
+ def load(self, oid, version):
+ if version <> '':
+ raise POSException.Unsupported, 'versions are not supported'
+ self._lock_acquire()
+ try:
+ # Get the current serial number for this object
+ serial = self._getCurrentSerial(oid)
+ if serial is None:
+ raise KeyError, 'Object does not exist: %r' % oid
+ # Get this revision's pickle data
+ return self._pickles[oid+serial], serial
finally:
self._lock_release()
+
+ def modifiedInVersion(self, oid):
+ # So BaseStorage.getSerial() just works. Note that this storage
+ # doesn't support versions.
+ return ''
=== Removed File ZODB3/bsddb3Storage/bsddb3Storage/CommitLog.py ===