[Zodb-checkins] CVS: ZODB3/BDBStorage - BDBFullStorage.py:1.44.4.2 BDBMinimalStorage.py:1.12.6.1 BerkeleyBase.py:1.19.4.1 __init__.py:1.8.6.1

Barry Warsaw barry@wooz.org
Tue, 7 Jan 2003 14:38:57 -0500


Update of /cvs-repository/ZODB3/BDBStorage
In directory cvs.zope.org:/tmp/cvs-serv24443/BDBStorage

Modified Files:
      Tag: ZODB3-3_1-branch
	BDBFullStorage.py BDBMinimalStorage.py BerkeleyBase.py 
	__init__.py 
Log Message:
Sync'ing with the trunk for BDBStorage




=== ZODB3/BDBStorage/BDBFullStorage.py 1.44.4.1 => 1.44.4.2 === (2800/2900 lines abridged)
--- ZODB3/BDBStorage/BDBFullStorage.py:1.44.4.1	Fri Oct 18 16:49:50 2002
+++ ZODB3/BDBStorage/BDBFullStorage.py	Tue Jan  7 14:38:52 2003
@@ -13,53 +13,32 @@
 ##############################################################################
 
 """Berkeley storage with full undo and versioning support.
-
-See Minimal.py for an implementation of Berkeley storage that does not support
-undo or versioning.
 """
 
 __version__ = '$Revision$'.split()[-2:][0]
 
-import sys
-import struct
 import time
-
-from cPickle import loads, Pickler
-Pickler = Pickler()
-Pickler.fast = 1 # Don't use a memo
-fast_pickle_dumps = Pickler.dump
-del Pickler
-
-# This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
-# http://pybsddb.sourceforge.net
-from bsddb3 import db
+import cPickle as pickle
+from struct import pack, unpack
 
 from ZODB import POSException
 from ZODB.utils import p64, U64
 from ZODB.referencesf import referencesf
 from ZODB.TimeStamp import TimeStamp
 from ZODB.ConflictResolution import ConflictResolvingStorage, ResolvedSerial
-import ThreadLock
 
-# BerkeleyBase.BerkeleyBase class provides some common functionality for both
-# the Full and Minimal implementations.  It in turn inherits from
-# ZODB.BaseStorage.BaseStorage which itself provides some common storage
-# functionality.
-from BerkeleyBase import BerkeleyBase
-from CommitLog import FullLog
-
-# Flags for transaction status in the transaction metadata table.  You can
-# only undo back to the last pack, and any transactions before the pack time
-# get marked with the PROTECTED_TRANSACTION flag.  An attempt to undo past a
-# PROTECTED_TRANSACTION will raise an POSException.UndoError.  By default,
-# transactions are marked with the UNDOABLE_TRANSACTION status flag.
-UNDOABLE_TRANSACTION = 'Y'
-PROTECTED_TRANSACTION = 'N'

[-=- -=- -=- 2800 lines omitted -=- -=- -=-]

+        if packedp:
             self.status = 'p'
+        else:
+            self.status = ' '
         self.user = user
         self.description = desc
-        self._extension = ext
+        try:
+            self._extension = pickle.loads(ext)
+        except EOFError:
+            self._extension = {}
         # Internal pointer
         self._oids = self._storage._alltxnoids(self.tid)
         # To make .pop() more efficient
         self._oids.reverse()
 
-    def __getitem__(self, i):
+    def next(self):
         """Return the ith item in the sequence of record data.
 
         Items must be accessed sequentially, and are instances of Record.  An
@@ -1612,3 +1869,28 @@
         self.version = version
         self.data = data
         self.data_txn = data_txn
+
+
+
+class _Autopack(_WorkThread):
+    NAME = 'autopacking'
+
+    def __init__(self, storage, event,
+                 frequency, packtime, classicpack,
+                 lastpacktime):
+        _WorkThread.__init__(self, storage, event, frequency)
+        self._packtime = packtime
+        self._classicpack = classicpack
+        # Bookkeeping
+        self._lastclassic = 0
+
+    def _dowork(self):
+        # Should we do a classic pack this time?
+        if self._classicpack <= 0:
+            classicp = False
+        else:
+            v = (self._lastclassic + 1) % self._classicpack
+            self._lastclassic = v
+            classicp = not v
+        # Run the autopack phase
+        self._storage.autopack(time.time() - self._packtime, classicp)


=== ZODB3/BDBStorage/BDBMinimalStorage.py 1.12 => 1.12.6.1 === (631/731 lines abridged)
--- ZODB3/BDBStorage/BDBMinimalStorage.py:1.12	Tue Feb 12 17:33:09 2002
+++ ZODB3/BDBStorage/BDBMinimalStorage.py	Tue Jan  7 14:38:52 2003
@@ -2,247 +2,536 @@
 #
 # Copyright (c) 2001, 2002 Zope Corporation and Contributors.
 # All Rights Reserved.
-# 
+#
 # This software is subject to the provisions of the Zope Public License,
 # Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
 # FOR A PARTICULAR PURPOSE
-# 
+#
 ##############################################################################
 
 """Berkeley storage without undo or versioning.
-
-See Full.py for an implementation of Berkeley storage that does support undo
-and versioning.
 """
 
 __version__ = '$Revision$'[-2:][0]
 
-# This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
-# http://pybsddb.sourceforge.net.  It is compatible with release 3.0 of
-# PyBSDDB3.
-from bsddb3 import db
-
-# BerkeleyBase.BerkeleyBase class provides some common functionality for both
-# the Full and Minimal implementations.  It in turn inherits from
-# ZODB.BaseStorage.BaseStorage which itself provides some common storage
-# functionality.
-from BerkeleyBase import BerkeleyBase
-from CommitLog import PacklessLog
 from ZODB import POSException
-from ZODB import utils
+from ZODB.utils import p64, U64
+from ZODB.referencesf import referencesf
+from ZODB.ConflictResolution import ConflictResolvingStorage, ResolvedSerial
+
+from BDBStorage import db
+from BerkeleyBase import BerkeleyBase, PackStop, _WorkThread
+
+ABORT = 'A'
+COMMIT = 'C'
+PRESENT = 'X'
+ZERO = '\0'*8

[-=- -=- -=- 631 lines omitted -=- -=- -=-]

+                    data = rec[1]
+                    c.delete()
+                    rec = c.next()
+                    deltas = {}
+                    self._update(deltas, data, -1)
+                    for oid, delta in deltas.items():
+                        refcount = U64(self._refcounts.get(oid, ZERO)) + delta
+                        if refcount <= 0:
+                            self._oidqueue.append(oid, txn)
+                        else:
+                            self._refcounts.put(oid, p64(refcount), txn=txn)
+            finally:
+                c.close()
+            # We really do want this down here, since _decrefPickle() could
+            # add more items to the queue.
+            orec = self._oidqueue.consume(txn)
+        assert len(self._oidqueue) == 0
+
+    #
+    # Stuff we don't support
+    #
+
+    def supportsTransactionalUndo(self):
+        return False
+
+    def supportsUndo(self):
+        return False
+
+    def supportsVersions(self):
+        return False
+
+    # Don't implement these
+    #
+    # versionEmpty(self, version)
+    # versions(self, max=None)
+    # loadSerial(self, oid, serial)
+    # getSerial(self, oid)
+    # transactionalUndo(self, tid, transaction)
+    # undoLog(self, first=0, last=-20, filter=None)
+    # history(self, oid, version=None, size=1, filter=None)
+    # iterator(self, start=None, stop=None)
+
+
+
+class _Autopack(_WorkThread):
+    NAME = 'autopacking'
+
+    def _dowork(self):
+        # Run the autopack phase
+        self._storage.pack('ignored', referencesf)


=== ZODB3/BDBStorage/BerkeleyBase.py 1.19 => 1.19.4.1 === (485/585 lines abridged)
--- ZODB3/BDBStorage/BerkeleyBase.py:1.19	Tue Sep  3 14:07:30 2002
+++ ZODB3/BDBStorage/BerkeleyBase.py	Tue Jan  7 14:38:52 2003
@@ -2,105 +2,152 @@
 #
 # Copyright (c) 2001, 2002 Zope Corporation and Contributors.
 # All Rights Reserved.
-# 
+#
 # This software is subject to the provisions of the Zope Public License,
 # Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
 # FOR A PARTICULAR PURPOSE
-# 
+#
 ##############################################################################
 
 """Base class for BerkeleyStorage implementations.
 """
+__version__ = '$Revision$'.split()[-2:][0]
 
 import os
+import time
 import errno
+import shutil
+import threading
 from types import StringType
 
 # This uses the Dunn/Kuchling PyBSDDB v3 extension module available from
 # http://pybsddb.sourceforge.net
-from bsddb3 import db
+from BDBStorage import db
 
-# BaseStorage provides some common storage functionality.  It is derived from
-# UndoLogCompatible.UndoLogCompatible, which "[provides] backward
-# compatability with storages that have undoLog, but not undoInfo."
-#
-# BAW: I'm not entirely sure what that means, but the UndoLogCompatible
-# subclass provides one method:
-#
-# undoInfo(first, last, specification).  Unfortunately this method appears to
-# be undocumented.  Jeremy tells me it's still required though.
-#
-# BaseStorage provides primitives for lock acquisition and release,
-# abortVersion(), commitVersion() and a host of other methods, some of which
-# are overridden here, some of which are not.
-from ZODB import POSException
+# BaseStorage provides primitives for lock acquisition and release, and a host
+# of other methods, some of which are overridden here, some of which are not.

[-=- -=- -=- 485 lines omitted -=- -=- -=-]

+    try:
+        shutil.rmtree(envdir)
+    except OSError, e:
+        if e.errno <> errno.ENOENT:
+            raise
+
+
+
+class _WorkThread(threading.Thread):
+    NAME = 'worker'
+
+    def __init__(self, storage, event, checkinterval):
+        threading.Thread.__init__(self)
+        self._storage = storage
+        self._event = event
+        self._interval = checkinterval
+        # Bookkeeping.  _nextcheck is useful as a non-public interface aiding
+        # testing.  See test_autopack.py.
+        self._stop = False
+        self._nextcheck = checkinterval
+        # We don't want these threads to hold up process exit.  That could
+        # lead to corrupt databases, but recovery should ultimately save us.
+        self.setDaemon(True)
+
+    def run(self):
+        name = self.NAME
+        self._storage.log('%s thread started', name)
+        while not self._stop:
+            now = time.time()
+            if now >= self._nextcheck:
+                self._storage.log('running %s', name)
+                self._dowork()
+                # Recalculate `now' because _dowork() could have taken a
+                # while.  time.time() can be expensive, but oh well.
+                self._nextcheck = time.time() + self._interval
+            # Block w/ timeout on the shutdown event.
+            self._event.wait(self._interval)
+            self._stop = self._event.isSet()
+        self._storage.log('%s thread finished', name)
+
+    def _dowork(self):
+        pass
+
+
+
+class _Checkpoint(_WorkThread):
+    NAME = 'checkpointing'
+
+    def _dowork(self):
+        self._storage.docheckpoint()


=== ZODB3/BDBStorage/__init__.py 1.8 => 1.8.6.1 ===
--- ZODB3/BDBStorage/__init__.py:1.8	Mon Feb 11 18:40:43 2002
+++ ZODB3/BDBStorage/__init__.py	Tue Jan  7 14:38:52 2003
@@ -2,14 +2,36 @@
 #
 # Copyright (c) 2001, 2002 Zope Corporation and Contributors.
 # All Rights Reserved.
-# 
+#
 # This software is subject to the provisions of the Zope Public License,
 # Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
 # FOR A PARTICULAR PURPOSE
-# 
+#
 ##############################################################################
 
-__version__ = '1.0 beta 5'
+# Python 2.2 and earlier requires the pybsddb distutils package, but for
+# Python 2.3, we really want to use the standard bsddb package.  Also, we want
+# to set a flag that other modules can easily tests to see if this stuff is
+# available or not.  Python 2.2 and 2.3 has bool() but not Python 2.1.
+#
+# Get the pybsddb extension module from pybsddb.sourceforge.net and the
+# BerkeleyDB libraries from www.sleepycat.com.
+
+try:
+    bool
+except NameError:
+    def bool(x):
+        return not not x
+
+try:
+    from bsddb import _db as db
+except ImportError:
+    try:
+        from bsddb3 import db
+    except ImportError:
+        db = None
+
+is_available = bool(db)