[Zodb-checkins] CVS: ZODB3/BDBStorage - BDBFullStorage.py:1.44.4.2 BDBMinimalStorage.py:1.12.6.1 BerkeleyBase.py:1.19.4.1 __init__.py:1.8.6.1
Barry Warsaw
barry@wooz.org
Tue, 7 Jan 2003 14:38:57 -0500
Update of /cvs-repository/ZODB3/BDBStorage
In directory cvs.zope.org:/tmp/cvs-serv24443/BDBStorage
Modified Files:
Tag: ZODB3-3_1-branch
BDBFullStorage.py BDBMinimalStorage.py BerkeleyBase.py
__init__.py
Log Message:
Sync'ing with the trunk for BDBStorage
=== ZODB3/BDBStorage/BDBFullStorage.py 1.44.4.1 => 1.44.4.2 === (2800/2900 lines abridged)
--- ZODB3/BDBStorage/BDBFullStorage.py:1.44.4.1 Fri Oct 18 16:49:50 2002
+++ ZODB3/BDBStorage/BDBFullStorage.py Tue Jan 7 14:38:52 2003
@@ -13,53 +13,32 @@
##############################################################################
"""Berkeley storage with full undo and versioning support.
-
-See Minimal.py for an implementation of Berkeley storage that does not support
-undo or versioning.
"""
__version__ = '$Revision$'.split()[-2:][0]
-import sys
-import struct
import time
-
-from cPickle import loads, Pickler
-Pickler = Pickler()
-Pickler.fast = 1 # Don't use a memo
-fast_pickle_dumps = Pickler.dump
-del Pickler
-
-# This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
-# http://pybsddb.sourceforge.net
-from bsddb3 import db
+import cPickle as pickle
+from struct import pack, unpack
from ZODB import POSException
from ZODB.utils import p64, U64
from ZODB.referencesf import referencesf
from ZODB.TimeStamp import TimeStamp
from ZODB.ConflictResolution import ConflictResolvingStorage, ResolvedSerial
-import ThreadLock
-# BerkeleyBase.BerkeleyBase class provides some common functionality for both
-# the Full and Minimal implementations. It in turn inherits from
-# ZODB.BaseStorage.BaseStorage which itself provides some common storage
-# functionality.
-from BerkeleyBase import BerkeleyBase
-from CommitLog import FullLog
-
-# Flags for transaction status in the transaction metadata table. You can
-# only undo back to the last pack, and any transactions before the pack time
-# get marked with the PROTECTED_TRANSACTION flag. An attempt to undo past a
-# PROTECTED_TRANSACTION will raise an POSException.UndoError. By default,
-# transactions are marked with the UNDOABLE_TRANSACTION status flag.
-UNDOABLE_TRANSACTION = 'Y'
-PROTECTED_TRANSACTION = 'N'
[-=- -=- -=- 2800 lines omitted -=- -=- -=-]
+ if packedp:
self.status = 'p'
+ else:
+ self.status = ' '
self.user = user
self.description = desc
- self._extension = ext
+ try:
+ self._extension = pickle.loads(ext)
+ except EOFError:
+ self._extension = {}
# Internal pointer
self._oids = self._storage._alltxnoids(self.tid)
# To make .pop() more efficient
self._oids.reverse()
- def __getitem__(self, i):
+ def next(self):
"""Return the ith item in the sequence of record data.
Items must be accessed sequentially, and are instances of Record. An
@@ -1612,3 +1869,28 @@
self.version = version
self.data = data
self.data_txn = data_txn
+
+
+
+class _Autopack(_WorkThread):
+ NAME = 'autopacking'
+
+ def __init__(self, storage, event,
+ frequency, packtime, classicpack,
+ lastpacktime):
+ _WorkThread.__init__(self, storage, event, frequency)
+ self._packtime = packtime
+ self._classicpack = classicpack
+ # Bookkeeping
+ self._lastclassic = 0
+
+ def _dowork(self):
+ # Should we do a classic pack this time?
+ if self._classicpack <= 0:
+ classicp = False
+ else:
+ v = (self._lastclassic + 1) % self._classicpack
+ self._lastclassic = v
+ classicp = not v
+ # Run the autopack phase
+ self._storage.autopack(time.time() - self._packtime, classicp)
=== ZODB3/BDBStorage/BDBMinimalStorage.py 1.12 => 1.12.6.1 === (631/731 lines abridged)
--- ZODB3/BDBStorage/BDBMinimalStorage.py:1.12 Tue Feb 12 17:33:09 2002
+++ ZODB3/BDBStorage/BDBMinimalStorage.py Tue Jan 7 14:38:52 2003
@@ -2,247 +2,536 @@
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
-#
+#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
-#
+#
##############################################################################
"""Berkeley storage without undo or versioning.
-
-See Full.py for an implementation of Berkeley storage that does support undo
-and versioning.
"""
__version__ = '$Revision$'[-2:][0]
-# This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
-# http://pybsddb.sourceforge.net. It is compatible with release 3.0 of
-# PyBSDDB3.
-from bsddb3 import db
-
-# BerkeleyBase.BerkeleyBase class provides some common functionality for both
-# the Full and Minimal implementations. It in turn inherits from
-# ZODB.BaseStorage.BaseStorage which itself provides some common storage
-# functionality.
-from BerkeleyBase import BerkeleyBase
-from CommitLog import PacklessLog
from ZODB import POSException
-from ZODB import utils
+from ZODB.utils import p64, U64
+from ZODB.referencesf import referencesf
+from ZODB.ConflictResolution import ConflictResolvingStorage, ResolvedSerial
+
+from BDBStorage import db
+from BerkeleyBase import BerkeleyBase, PackStop, _WorkThread
+
+ABORT = 'A'
+COMMIT = 'C'
+PRESENT = 'X'
+ZERO = '\0'*8
[-=- -=- -=- 631 lines omitted -=- -=- -=-]
+ data = rec[1]
+ c.delete()
+ rec = c.next()
+ deltas = {}
+ self._update(deltas, data, -1)
+ for oid, delta in deltas.items():
+ refcount = U64(self._refcounts.get(oid, ZERO)) + delta
+ if refcount <= 0:
+ self._oidqueue.append(oid, txn)
+ else:
+ self._refcounts.put(oid, p64(refcount), txn=txn)
+ finally:
+ c.close()
+ # We really do want this down here, since _decrefPickle() could
+ # add more items to the queue.
+ orec = self._oidqueue.consume(txn)
+ assert len(self._oidqueue) == 0
+
+ #
+ # Stuff we don't support
+ #
+
+ def supportsTransactionalUndo(self):
+ return False
+
+ def supportsUndo(self):
+ return False
+
+ def supportsVersions(self):
+ return False
+
+ # Don't implement these
+ #
+ # versionEmpty(self, version)
+ # versions(self, max=None)
+ # loadSerial(self, oid, serial)
+ # getSerial(self, oid)
+ # transactionalUndo(self, tid, transaction)
+ # undoLog(self, first=0, last=-20, filter=None)
+ # history(self, oid, version=None, size=1, filter=None)
+ # iterator(self, start=None, stop=None)
+
+
+
+class _Autopack(_WorkThread):
+ NAME = 'autopacking'
+
+ def _dowork(self):
+ # Run the autopack phase
+ self._storage.pack('ignored', referencesf)
=== ZODB3/BDBStorage/BerkeleyBase.py 1.19 => 1.19.4.1 === (485/585 lines abridged)
--- ZODB3/BDBStorage/BerkeleyBase.py:1.19 Tue Sep 3 14:07:30 2002
+++ ZODB3/BDBStorage/BerkeleyBase.py Tue Jan 7 14:38:52 2003
@@ -2,105 +2,152 @@
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
-#
+#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
-#
+#
##############################################################################
"""Base class for BerkeleyStorage implementations.
"""
+__version__ = '$Revision$'.split()[-2:][0]
import os
+import time
import errno
+import shutil
+import threading
from types import StringType
# This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
# http://pybsddb.sourceforge.net
-from bsddb3 import db
+from BDBStorage import db
-# BaseStorage provides some common storage functionality. It is derived from
-# UndoLogCompatible.UndoLogCompatible, which "[provides] backward
-# compatability with storages that have undoLog, but not undoInfo."
-#
-# BAW: I'm not entirely sure what that means, but the UndoLogCompatible
-# subclass provides one method:
-#
-# undoInfo(first, last, specification). Unfortunately this method appears to
-# be undocumented. Jeremy tells me it's still required though.
-#
-# BaseStorage provides primitives for lock acquisition and release,
-# abortVersion(), commitVersion() and a host of other methods, some of which
-# are overridden here, some of which are not.
-from ZODB import POSException
+# BaseStorage provides primitives for lock acquisition and release, and a host
+# of other methods, some of which are overridden here, some of which are not.
[-=- -=- -=- 485 lines omitted -=- -=- -=-]
+ try:
+ shutil.rmtree(envdir)
+ except OSError, e:
+ if e.errno <> errno.ENOENT:
+ raise
+
+
+
+class _WorkThread(threading.Thread):
+ NAME = 'worker'
+
+ def __init__(self, storage, event, checkinterval):
+ threading.Thread.__init__(self)
+ self._storage = storage
+ self._event = event
+ self._interval = checkinterval
+ # Bookkeeping. _nextcheck is useful as a non-public interface aiding
+ # testing. See test_autopack.py.
+ self._stop = False
+ self._nextcheck = checkinterval
+ # We don't want these threads to hold up process exit. That could
+ # lead to corrupt databases, but recovery should ultimately save us.
+ self.setDaemon(True)
+
+ def run(self):
+ name = self.NAME
+ self._storage.log('%s thread started', name)
+ while not self._stop:
+ now = time.time()
+ if now >= self._nextcheck:
+ self._storage.log('running %s', name)
+ self._dowork()
+ # Recalculate `now' because _dowork() could have taken a
+ # while. time.time() can be expensive, but oh well.
+ self._nextcheck = time.time() + self._interval
+ # Block w/ timeout on the shutdown event.
+ self._event.wait(self._interval)
+ self._stop = self._event.isSet()
+ self._storage.log('%s thread finished', name)
+
+ def _dowork(self):
+ pass
+
+
+
+class _Checkpoint(_WorkThread):
+ NAME = 'checkpointing'
+
+ def _dowork(self):
+ self._storage.docheckpoint()
=== ZODB3/BDBStorage/__init__.py 1.8 => 1.8.6.1 ===
--- ZODB3/BDBStorage/__init__.py:1.8 Mon Feb 11 18:40:43 2002
+++ ZODB3/BDBStorage/__init__.py Tue Jan 7 14:38:52 2003
@@ -2,14 +2,36 @@
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
-#
+#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
-#
+#
##############################################################################
-__version__ = '1.0 beta 5'
+# Python 2.2 and earlier requires the pybsddb distutils package, but for
+# Python 2.3, we really want to use the standard bsddb package. Also, we want
+# to set a flag that other modules can easily tests to see if this stuff is
+# available or not. Python 2.2 and 2.3 has bool() but not Python 2.1.
+#
+# Get the pybsddb extension module from pybsddb.sourceforge.net and the
+# BerkeleyDB libraries from www.sleepycat.com.
+
+try:
+ bool
+except NameError:
+ def bool(x):
+ return not not x
+
+try:
+ from bsddb import _db as db
+except ImportError:
+ try:
+ from bsddb3 import db
+ except ImportError:
+ db = None
+
+is_available = bool(db)