[Zope3-checkins] CVS: ZODB4/src/zodb - transact.py:1.2 component.xml:1.2 serialize.py:1.22 dbdump.py:1.4 connection.py:1.35 conflict.py:1.15 config.py:1.4
Jeremy Hylton
jeremy@zope.com
Thu, 19 Jun 2003 17:41:41 -0400
Update of /cvs-repository/ZODB4/src/zodb
In directory cvs.zope.org:/tmp/cvs-serv15960/src/zodb
Modified Files:
serialize.py dbdump.py connection.py conflict.py config.py
Added Files:
transact.py component.xml
Log Message:
Merge ZODB3-2-merge branch to the head.
This completes the porting of bug fixes and random improvements from
ZODB 3.2 to ZODB 4.
=== ZODB4/src/zodb/transact.py 1.1 => 1.2 ===
--- /dev/null Thu Jun 19 17:41:41 2003
+++ ZODB4/src/zodb/transact.py Thu Jun 19 17:41:10 2003
@@ -0,0 +1,59 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Tools to simplify transactions within applications."""
+
+from transaction import get_transaction
+from zodb.interfaces import ReadConflictError, ConflictError
+
+def _commit(note):
+ t = get_transaction()
+ if note:
+ t.note(note)
+ t.commit()
+
+def transact(f, note=None, retries=5):
+ """Returns transactional version of function argument f.
+
+ Higher-order function that converts a regular function into
+ a transactional function. The transactional function will
+ retry up to retries time before giving up. If note, it will
+ be added to the transaction metadata when it commits.
+
+ The retries occur on ConflictErrors. If some other
+ TransactionError occurs, the transaction will not be retried.
+ """
+
+ # XXX deal with ZEO disconnected errors?
+
+ def g(*args, **kwargs):
+ n = retries
+ while n:
+ n -= 1
+ try:
+ r = f(*args, **kwargs)
+ except ReadConflictError, msg:
+ get_transaction().abort()
+ if not n:
+ raise
+ continue
+ try:
+ _commit(note)
+ except ConflictError, msg:
+ get_transaction().abort()
+ if not n:
+ raise
+ continue
+ return r
+ raise RuntimeError, "couldn't commit transaction"
+ return g
=== ZODB4/src/zodb/component.xml 1.1 => 1.2 ===
--- /dev/null Thu Jun 19 17:41:41 2003
+++ ZODB4/src/zodb/component.xml Thu Jun 19 17:41:10 2003
@@ -0,0 +1,162 @@
+<component prefix="zodb.config">
+
+ <!-- XXX needs descriptions for everything -->
+
+ <abstracttype name="zodb.storage"/>
+ <abstracttype name="zodb.database"/>
+
+ <sectiontype name="filestorage" datatype=".FileStorage"
+ implements="zodb.storage">
+ <key name="path" required="yes">
+ <description>
+ Path name to the main storage file. The names for
+ supplemental files, including index and lock files, will be
+ computed from this.
+ </description>
+ </key>
+ <key name="create" datatype="boolean" default="false">
+ <description>
+ Flag that indicates whether the storage should be truncated if
+ it already exists.
+ </description>
+ </key>
+ <key name="read-only" datatype="boolean" default="false">
+ <description>
+ If true, only reads may be executed against the storage. Note
+ that the "pack" operation is not considered a write operation
+ and is still allowed on a read-only filestorage.
+ </description>
+ </key>
+ <key name="quota" datatype="byte-size">
+ <description>
+ Maximum allowed size of the storage file. Operations which
+ would cause the size of the storage to exceed the quota will
+ result in a zodb.FileStorage.FileStorageQuotaError being
+ raised.
+ </description>
+ </key>
+ </sectiontype>
+
+ <sectiontype name="mappingstorage" datatype=".MappingStorage"
+ implements="zodb.storage">
+ <key name="name" default="Mapping Storage"/>
+ </sectiontype>
+
+ <!-- The BDB storages probably need to be revised somewhat still.
+ The extension relationship seems a little odd.
+ -->
+ <sectiontype name="fullstorage" datatype=".BDBFullStorage"
+ implements="zodb.storage">
+ <key name="name" required="yes" />
+ <key name="envdir" />
+ <key name="interval" datatype="time-interval" default="2m" />
+ <key name="kbyte" datatype="integer" default="0" />
+ <key name="min" datatype="integer" default="0" />
+ <key name="logdir" />
+ <key name="cachesize" datatype="byte-size" default="128MB" />
+ <key name="frequency" datatype="time-interval" default="0" />
+ <key name="packtime" datatype="time-interval" default="4h" />
+ <key name="gcpack" datatype="integer" default="0" />
+ <key name="read-only" datatype="boolean" default="off"/>
+ </sectiontype>
+
+ <sectiontype name="minimalstorage" datatype=".BDBMinimalStorage"
+ implements="zodb.storage" extends="fullstorage"/>
+
+ <sectiontype name="zeoclient" datatype=".ZEOClient"
+ implements="zodb.storage">
+ <multikey name="server" datatype="socket-address" required="yes"/>
+ <key name="storage" default="1">
+ <description>
+ The name of the storage that the client wants to use. If the
+ ZEO server serves more than one storage, the client selects
+ the storage it wants to use by name. The default name is '1',
+ which is also the default name for the ZEO server.
+ </description>
+ </key>
+ <key name="cache-size" datatype="integer" default="20000000">
+ <description>
+ The maximum size of the client cache, in bytes.
+ </description>
+ </key>
+ <key name="name" default="">
+ <description>
+ The storage name. If unspecified, the address of the server
+ will be used as the name.
+ </description>
+ </key>
+ <key name="client">
+ <description>
+ Enables persistent cache files. The string passed here is
+ used to construct the cache filenames. If it is not
+ specified, the client creates a temporary cache that will
+ only be used by the current object.
+ </description>
+ </key>
+ <key name="var">
+ <description>
+ The directory where persistent cache files are stored. By
+ default cache files, if they are persistent, are stored in
+ the current directory.
+ </description>
+ </key>
+ <key name="min-disconnect-poll" datatype="integer" default="5">
+ <description>
+ The minimum delay in seconds between attempts to connect to
+ the server, in seconds. Defaults to 5 seconds.
+ </description>
+ </key>
+ <key name="max-disconnect-poll" datatype="integer" default="300">
+ <description>
+ The maximum delay in seconds between attempts to connect to
+ the server, in seconds. Defaults to 300 seconds.
+ </description>
+ </key>
+ <key name="wait" datatype="boolean" default="on">
+ <description>
+ A boolean indicating whether the constructor should wait
+ for the client to connect to the server and verify the cache
+ before returning. The default is true.
+ </description>
+ </key>
+ <key name="read-only" datatype="boolean" default="off">
+ <description>
+ A flag indicating whether this should be a read-only storage,
+ defaulting to false (i.e. writing is allowed by default).
+ </description>
+ </key>
+ <key name="read-only-fallback" datatype="boolean" default="off">
+ <description>
+ A flag indicating whether a read-only remote storage should be
+ acceptable as a fallback when no writable storages are
+ available. Defaults to false. At most one of read_only and
+ read_only_fallback should be true.
+ </description>
+ </key>
+ <key name="realm" required="no">
+ <description>
+ The authentication realm of the server. Some authentication
+ schemes use a realm to identify the logic set of usernames
+ that are accepted by this server.
+ </description>
+ </key>
+ </sectiontype>
+
+ <sectiontype name="demostorage" datatype=".DemoStorage"
+ implements="zodb.storage">
+ <key name="name" default="Demo Storage"/>
+ <section type="zodb.storage" name="*" attribute="base"/>
+ <key name="quota" datatype="integer"/>
+ </sectiontype>
+
+
+ <sectiontype name="zodb" datatype=".ZODBDatabase"
+ implements="zodb.database">
+ <section type="zodb.storage" name="*" attribute="storage"/>
+ <key name="cache-size" datatype="integer" default="5000"/>
+ <key name="pool-size" datatype="integer" default="7"/>
+ <key name="version-pool-size" datatype="integer" default="3"/>
+ <key name="version-cache-size" datatype="integer" default="100"/>
+ </sectiontype>
+
+</component>
=== ZODB4/src/zodb/serialize.py 1.21 => 1.22 ===
--- ZODB4/src/zodb/serialize.py:1.21 Tue May 20 15:07:24 2003
+++ ZODB4/src/zodb/serialize.py Thu Jun 19 17:41:10 2003
@@ -218,8 +218,8 @@
def getClassName(self, pickle):
unpickler = self._get_unpickler(pickle)
- module, classname, newargs = unpickler.load()
- return "%s.%s" % (module, classname)
+ cls, newargs = unpickler.load()
+ return cls.__name__
def getGhost(self, pickle):
unpickler = self._get_unpickler(pickle)
=== ZODB4/src/zodb/dbdump.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/dbdump.py:1.3 Mon Dec 30 16:40:30 2002
+++ ZODB4/src/zodb/dbdump.py Thu Jun 19 17:41:10 2003
@@ -39,11 +39,13 @@
for rec in trans:
if rec.data is None:
fullclass = "undo or abort of object creation"
+ size = 0
else:
# Any object reader will do
reader = SimpleObjectReader()
fullclass = reader.getClassName(rec.data)
dig = md5.new(rec.data).hexdigest()
+ size = len(rec.data)
# special case for testing purposes
if fullclass == "zodb.tests.minpo.MinPO":
obj = zodb_unpickle(rec.data)
@@ -52,8 +54,8 @@
version = "version=%s " % rec.version
else:
version = ''
- print >> outp, " data #%05d oid=%016x %sclass=%s" % \
- (j, u64(rec.oid), version, fullclass)
+ print >> outp, " data #%05d oid=%016x %sclass=%s size=%d" % \
+ (j, u64(rec.oid), version, fullclass, size)
j += 1
print >> outp
i += 1
=== ZODB4/src/zodb/connection.py 1.34 => 1.35 ===
--- ZODB4/src/zodb/connection.py:1.34 Fri Jun 6 11:24:19 2003
+++ ZODB4/src/zodb/connection.py Thu Jun 19 17:41:10 2003
@@ -41,6 +41,7 @@
import struct
import tempfile
import threading
+from types import StringType
from zope.interface import implements
@@ -396,51 +397,47 @@
# Now is a good time to collect some garbage
self._cache.shrink()
- def _handle_serial(self, store_return, oid=None, change=True):
+ def _handle_serial(self, store_return, oid=None, change=1):
"""Handle the returns from store() and tpc_vote() calls."""
- # XXX We could simplify the storage interface if ZEO would
- # raise the exception itself rather than passing it into the
- # connection.
-
# These calls can return different types depending on whether
# ZEO is used. ZEO uses asynchronous returns that may be
# returned in batches by the ClientStorage. ZEO1 can also
# return an exception object and expect that the Connection
# will raise the exception.
- # When _commit_sub() exceutes a store, there is no need to
+ # When commit_sub() exceutes a store, there is no need to
# update the _p_changed flag, because the subtransaction
- # tpcVote() calls already did this. The change=1 argument
- # exists to allow _commit_sub() to avoid setting the flag
+ # tpc_vote() calls already did this. The change=1 argument
+ # exists to allow commit_sub() to avoid setting the flag
# again.
+
+ # When conflict resolution occurs, the object state held by
+ # the connection does not match what is written to the
+ # database. Invalidate the object here to guarantee that
+ # the new state is read the next time the object is used.
+
if not store_return:
return
- if isinstance(store_return, str):
+ if isinstance(store_return, StringType):
assert oid is not None
- serial = store_return
- obj = self._cache.get(oid)
- if obj is None:
- return
- if serial == ResolvedSerial:
- obj._p_deactivate()
- else:
- if change:
- obj._p_changed = 0
- obj._p_serial = serial
+ self._handle_one_serial(oid, store_return, change)
else:
for oid, serial in store_return:
- if not isinstance(serial, str):
- raise serial
- obj = self._cache.get(oid)
- if obj is None:
- continue
- if serial == ResolvedSerial:
- obj._p_deactivate()
- else:
- if change:
- obj._p_changed = 0
- obj._p_serial = serial
+ self._handle_one_serial(oid, serial, change)
+
+ def _handle_one_serial(self, oid, serial, change=1):
+ if not isinstance(serial, StringType):
+ raise serial
+ obj = self._cache.get(oid, None)
+ if obj is None:
+ return
+ if serial == ResolvedSerial:
+ obj._p_deactivate(force=True)
+ else:
+ if change:
+ obj._p_changed = False
+ obj._p_serial = serial
def _objcommit(self, obj, transaction):
oid = obj._p_oid
=== ZODB4/src/zodb/conflict.py 1.14 => 1.15 ===
--- ZODB4/src/zodb/conflict.py:1.14 Thu May 1 15:34:58 2003
+++ ZODB4/src/zodb/conflict.py Thu Jun 19 17:41:10 2003
@@ -23,6 +23,8 @@
from zodb.interfaces import ConflictError
from zodb.serialize import BaseObjectReader, ObjectWriter
+from zodb.interfaces import _fmt_oid
+from zodb.utils import u64
ResolvedSerial = "rs"
@@ -167,13 +169,6 @@
return None
newstate = reader.getState(newpickle)
- # XXX Using loadSerial() ties conflict resolution to the IUndoStorage
- # interface. This is a bad thing for non-versioning storages like
- # BDBMinimalStorage and MemoryMinimalStorage because they don't
- # support undo, and thus do not implement IUndoStorage. IUndoStorage
- # is the interface that defines loadSerial(). Hmm, maybe we should
- # move loadSerial() out of that interface?
-
p = self._storage.loadSerial(oid, oldSerial)
try:
old = reader.getState(p)
@@ -181,7 +176,13 @@
logging.warn("CR: Error loading object: %s", err)
return None
if committedData is None:
- committedData = self._storage.loadSerial(oid, committedSerial)
+ try:
+ committedData = self._storage.loadSerial(oid, committedSerial)
+ except KeyError:
+ logging.debug("CR: Could not load committed state "
+ "oid=%s serial=%s" % (_fmt_oid(oid),
+ u64(committedSerial)))
+ return None
try:
committed = reader.getState(committedData)
except (EOFError, PicklingError), err:
=== ZODB4/src/zodb/config.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/config.py:1.3 Wed Apr 9 17:15:16 2003
+++ ZODB4/src/zodb/config.py Thu Jun 19 17:41:10 2003
@@ -11,182 +11,162 @@
# FOR A PARTICULAR PURPOSE
#
##############################################################################
-"""Default storage types.
+"""Open database and storage from a configuration.
-Adapted from DBTab/StorageTypes.py.
-"""
+$Id$"""
-import re
+import os
+import StringIO
-from ZConfig.Config import asBoolean
+import ZConfig
+import zodb.db
-def convertFileStorageArgs(quota=None, stop=None, **kw):
- if kw.has_key('name'):
- # FileStorage doesn't accept a 'name' arg
- del kw['name']
- if quota is not None:
- kw['quota'] = long(quota) or None
- if stop is not None:
- stop = long(stop)
- if not stop:
- stop = None
- else:
- from zodb.utils import p64
- stop = p64(stop)
- kw['stop'] = stop
-
- # Boolean args
- for name in (
- 'create', 'read_only'
- ):
- if kw.has_key(name):
- kw[name] = asBoolean(kw[name])
-
- return kw
-
-
-# Match URLs of the form 'zeo://zope.example.com:1234'
-zeo_url_re = re.compile('zeo:/*(?P<host>[A-Za-z0-9\.-]+):(?P<port>[0-9]+)')
-
-def convertAddresses(s):
- # Allow multiple addresses using semicolons as a split character.
- res = []
- for a in s.split(';'):
- a = a.strip()
- if a:
- mo = zeo_url_re.match(a)
- if mo is not None:
- # ZEO URL
- host, port = mo.groups()
- res.append((host, int(port)))
- else:
- # Socket file
- res.append(a)
- return res
-
-
-def convertClientStorageArgs(addr=None, **kw):
- if addr is None:
- raise RuntimeError, 'An addr parameter is required for ClientStorage.'
- kw['addr'] = convertAddresses(addr)
-
- # Integer args
- for name in (
- 'cache_size', 'min_disconnect_poll', 'max_disconnect_poll',
- ):
- if kw.has_key(name):
- kw[name] = int(kw[name])
-
- # Boolean args
- for name in (
- 'wait', 'read_only', 'read_only_fallback',
- ):
- if kw.has_key(name):
- kw[name] = asBoolean(kw[name])
-
- # The 'client' parameter must be None to be false. Yuck.
- if kw.has_key('client') and not kw['client']:
- kw['client'] = None
-
- return kw
-
-
-# Currently unused
-def convertBDBStorageArgs(**kw):
- from zodb.storage.base import BerkeleyConfig
- config = BerkeleyConfig()
- for name in dir(BerkeleyConfig):
- if name.startswith('_'):
- continue
- val = kw.get(name)
- if val is not None:
- if name == 'read_only':
- val = asBoolean(val)
- elif name != 'logdir':
- val = int(val)
- setattr(config, name, val)
- del kw[name]
- # XXX: Nobody ever passes in env
- assert not kw.has_key('env')
- kw['config'] = config
- return kw
-
-
-storage_types = {
- # A mapping from "type" (i.e. class) to 2-tuple of (module, converter).
- # converter may be None for no conversion necessary. type and module are
- # both strings, and module should be the dotted path for use in an
- # __import__().
- 'FileStorage' : ('zodb.storage.file', convertFileStorageArgs),
- 'MappingStorage' : ('zodb.storage.mapping', None),
- 'ClientStorage' : ('zodb.zeo.client', convertClientStorageArgs),
- 'BDBFullStorage' : ('zodb.storage.bdbfull', convertBDBStorageArgs),
- 'BDBMinimalStorage' : ('zodb.storage.bdbminimal', convertBDBStorageArgs),
- 'MemoryFullStorage' : ('zodb.storage.memory', convertBDBStorageArgs),
- 'MemoryMinimalStorage': ('zodb.storage.memory', convertBDBStorageArgs),
- }
-
-
-"""Higher-level support for configuring storages.
-
-Storages are configured a la DBTab.
-
-A storage section has the form
-
- <Storage Name (dependent)>
- # For example
- type FileStorage
- file_name var/Data.fs
- read_only 1
- </Storage>
-
-where Name and (dependent) are optional. Once you have retrieved the
-section object (probably with getSection("Storage", name), the
-function creatStorage() in this module will create the storage object
-for you.
-"""
-
-
-
-def createStorage(section):
- """Create a storage specified by a configuration section."""
- klass, args = getStorageInfo(section)
- return klass(**args)
-
-def getStorageInfo(section):
- """Extract a storage description from a configuration section.
-
- Return a tuple (klass, args) where klass is the storage class and
- args is a dictionary of keyword arguments. To create the storage,
- call klass(**args).
+db_schema_path = os.path.join(zodb.__path__[0], "config.xml")
+_db_schema = None
+
+s_schema_path = os.path.join(zodb.__path__[0], "storage.xml")
+_s_schema = None
+
+def getDbSchema():
+ global _db_schema
+ if _db_schema is None:
+ _db_schema = ZConfig.loadSchema(db_schema_path)
+ return _db_schema
+
+def getStorageSchema():
+ global _s_schema
+ if _s_schema is None:
+ _s_schema = ZConfig.loadSchema(s_schema_path)
+ return _s_schema
+
+def databaseFromString(s):
+ return databaseFromFile(StringIO.StringIO(s))
+
+def databaseFromFile(f):
+ config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
+ return databaseFromConfig(config.database)
+
+def databaseFromURL(url):
+ config, handler = ZConfig.loadConfig(getDbSchema(), url)
+ return databaseFromConfig(config.database)
+
+def databaseFromConfig(section):
+ return section.open()
+
+def storageFromString(s):
+ return storageFromFile(StringIO.StringIO(s))
+
+def storageFromFile(f):
+ config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
+ return storageFromConfig(config.storage)
+
+def storageFromURL(url):
+ config, handler = ZConfig.loadConfig(getStorageSchema(), url)
+ return storageFromConfig(config.storage)
+
+def storageFromConfig(section):
+ return section.open()
+
+
+class BaseConfig:
+ """Object representing a configured storage or database.
+
+ Methods:
+
+ open() -- open and return the configured object
+
+ Attributes:
+
+ name -- name of the storage
- Adapted from DatabaseFactory.setStorageParams() in DBTab.py.
"""
- type = section.get("type")
- if not type:
- raise RuntimeError, "A storage type is required"
- module = None
- pos = type.rfind(".")
- if pos >= 0:
- # Specified the module
- module, type = type[:pos], type[pos+1:]
- converter = None
- if not module:
- # Use a default module and argument converter.
- info = storage_types.get(type)
- if not info:
- raise RuntimeError, "Unknown storage type: %s" % type
- module, converter = info
- m = __import__(module, {}, {}, [type])
- klass = getattr(m, type)
-
- args = {}
- if section.name:
- args["name"] = section.name
- for key in section.keys():
- if key.lower() != "type":
- args[key] = section.get(key)
- if converter is not None:
- args = converter(**args)
- return (klass, args)
+
+ def __init__(self, config):
+ self.config = config
+ self.name = config.getSectionName()
+
+ def open(self):
+ """Open and return the storage object."""
+ raise NotImplementedError
+
+class ZODBDatabase(BaseConfig):
+
+ def open(self):
+ section = self.config
+ return zodb.db.DB(section.storage.open(),
+ pool_size=section.pool_size,
+ cache_size=section.cache_size,
+ version_pool_size=section.version_pool_size,
+ version_cache_size=section.version_cache_size)
+
+class MappingStorage(BaseConfig):
+
+ def open(self):
+ from zodb.storage.mapping import MappingStorage
+ return MappingStorage(self.config.name)
+
+class DemoStorage(BaseConfig):
+
+ def open(self):
+ from zodb.storage.demo import DemoStorage
+ if self.config.base:
+ base = self.config.base.open()
+ else:
+ base = None
+ return DemoStorage(self.config.name,
+ base=base,
+ quota=self.config.quota)
+
+class FileStorage(BaseConfig):
+
+ def open(self):
+ from zodb.storage.file import FileStorage
+ return FileStorage(self.config.path,
+ create=self.config.create,
+ read_only=self.config.read_only,
+ quota=self.config.quota)
+
+class ZEOClient(BaseConfig):
+
+ def open(self):
+ from zodb.zeo.client import ClientStorage
+ # config.server is a multikey of socket-address values
+ # where the value is a socket family, address tuple.
+ L = [server.address for server in self.config.server]
+ return ClientStorage(
+ L,
+ storage=self.config.storage,
+ cache_size=self.config.cache_size,
+ name=self.config.name,
+ client=self.config.client,
+ var=self.config.var,
+ min_disconnect_poll=self.config.min_disconnect_poll,
+ max_disconnect_poll=self.config.max_disconnect_poll,
+ wait=self.config.wait,
+ read_only=self.config.read_only,
+ read_only_fallback=self.config.read_only_fallback)
+
+class BDBStorage(BaseConfig):
+
+ def open(self):
+ from zodb.storage.base import BerkeleyConfig
+ storageclass = self.get_storageclass()
+ bconf = BerkeleyConfig()
+ for name in dir(BerkeleyConfig):
+ if name.startswith('_'):
+ continue
+ setattr(bconf, name, getattr(self.config, name))
+ return storageclass(self.config.name, config=bconf)
+
+class BDBMinimalStorage(BDBStorage):
+
+ def get_storageclass(self):
+ from zodb.storage.bdbminimal import BDBMinimalStorage
+ return BDBMinimalStorage
+
+class BDBFullStorage(BDBStorage):
+
+ def get_storageclass(self):
+ from zodb.storage.bdbfull import BDBFullStorage
+ return BDBFullStorage