[Zope3-checkins] CVS: ZODB4/src/zodb - component.xml:1.1.2.1 serialize.py:1.21.6.1 dbdump.py:1.3.44.1 connection.py:1.34.2.1 config.py:1.3.20.1
Jeremy Hylton
jeremy@zope.com
Tue, 17 Jun 2003 17:59:56 -0400
Update of /cvs-repository/ZODB4/src/zodb
In directory cvs.zope.org:/tmp/cvs-serv10995/src/zodb
Modified Files:
Tag: ZODB3-2-merge
serialize.py dbdump.py connection.py config.py
Added Files:
Tag: ZODB3-2-merge
component.xml
Log Message:
Checkpoint progress merging ZODB 3.2 features and fixes into ZODB4.
=== Added File ZODB4/src/zodb/component.xml ===
<component prefix="zodb.config">
<!-- XXX needs descriptions for everything -->
<abstracttype name="zodb.storage"/>
<abstracttype name="zodb.database"/>
<sectiontype name="filestorage" datatype=".FileStorage"
implements="zodb.storage">
<key name="path" required="yes">
<description>
Path name to the main storage file. The names for
supplemental files, including index and lock files, will be
computed from this.
</description>
</key>
<key name="create" datatype="boolean" default="false">
<description>
Flag that indicates whether the storage should be truncated if
it already exists.
</description>
</key>
<key name="read-only" datatype="boolean" default="false">
<description>
If true, only reads may be executed against the storage. Note
that the "pack" operation is not considered a write operation
and is still allowed on a read-only filestorage.
</description>
</key>
<key name="quota" datatype="byte-size">
<description>
Maximum allowed size of the storage file. Operations which
would cause the size of the storage to exceed the quota will
result in a zodb.FileStorage.FileStorageQuotaError being
raised.
</description>
</key>
</sectiontype>
<sectiontype name="mappingstorage" datatype=".MappingStorage"
implements="zodb.storage">
<key name="name" default="Mapping Storage"/>
</sectiontype>
<!-- The BDB storages probably need to be revised somewhat still.
The extension relationship seems a little odd.
-->
<sectiontype name="fullstorage" datatype=".BDBFullStorage"
implements="zodb.storage">
<key name="name" required="yes" />
<key name="interval" datatype="time-interval" default="2m" />
<key name="kbyte" datatype="integer" default="0" />
<key name="min" datatype="integer" default="0" />
<key name="logdir" />
<key name="cachesize" datatype="byte-size" default="128MB" />
<key name="frequency" datatype="time-interval" default="0" />
<key name="packtime" datatype="time-interval" default="4h" />
<key name="classicpack" datatype="integer" default="0" />
<key name="read-only" datatype="boolean" default="off"/>
</sectiontype>
<sectiontype name="minimalstorage" datatype=".BDBMinimalStorage"
implements="zodb.storage" extends="fullstorage"/>
<sectiontype name="zeoclient" datatype=".ZEOClient"
implements="zodb.storage">
<multikey name="server" datatype="socket-address" required="yes"/>
<key name="storage" default="1">
<description>
The name of the storage that the client wants to use. If the
ZEO server serves more than one storage, the client selects
the storage it wants to use by name. The default name is '1',
which is also the default name for the ZEO server.
</description>
</key>
<key name="cache-size" datatype="integer" default="20000000">
<description>
The maximum size of the client cache, in bytes.
</description>
</key>
<key name="name" default="">
<description>
The storage name. If unspecified, the address of the server
will be used as the name.
</description>
</key>
<key name="client">
<description>
Enables persistent cache files. The string passed here is
used to construct the cache filenames. If it is not
specified, the client creates a temporary cache that will
only be used by the current object.
</description>
</key>
<key name="var">
<description>
The directory where persistent cache files are stored. By
default cache files, if they are persistent, are stored in
the current directory.
</description>
</key>
<key name="min-disconnect-poll" datatype="integer" default="5">
<description>
The minimum delay in seconds between attempts to connect to
the server, in seconds. Defaults to 5 seconds.
</description>
</key>
<key name="max-disconnect-poll" datatype="integer" default="300">
<description>
The maximum delay in seconds between attempts to connect to
the server, in seconds. Defaults to 300 seconds.
</description>
</key>
<key name="wait" datatype="boolean" default="on">
<description>
A boolean indicating whether the constructor should wait
for the client to connect to the server and verify the cache
before returning. The default is true.
</description>
</key>
<key name="read-only" datatype="boolean" default="off">
<description>
A flag indicating whether this should be a read-only storage,
defaulting to false (i.e. writing is allowed by default).
</description>
</key>
<key name="read-only-fallback" datatype="boolean" default="off">
<description>
A flag indicating whether a read-only remote storage should be
acceptable as a fallback when no writable storages are
available. Defaults to false. At most one of read_only and
read_only_fallback should be true.
</description>
</key>
<key name="realm" required="no">
<description>
The authentication realm of the server. Some authentication
schemes use a realm to identify the logic set of usernames
that are accepted by this server.
</description>
</key>
</sectiontype>
<sectiontype name="demostorage" datatype=".DemoStorage"
implements="zodb.storage">
<key name="name" default="Demo Storage"/>
<section type="zodb.storage" name="*" attribute="base"/>
<key name="quota" datatype="integer"/>
</sectiontype>
<sectiontype name="zodb" datatype=".ZODBDatabase"
implements="zodb.database">
<section type="zodb.storage" name="*" attribute="storage"/>
<key name="cache-size" datatype="integer" default="5000"/>
<key name="pool-size" datatype="integer" default="7"/>
<key name="version-pool-size" datatype="integer" default="3"/>
<key name="version-cache-size" datatype="integer" default="100"/>
</sectiontype>
</component>
=== ZODB4/src/zodb/serialize.py 1.21 => 1.21.6.1 ===
--- ZODB4/src/zodb/serialize.py:1.21 Tue May 20 15:07:24 2003
+++ ZODB4/src/zodb/serialize.py Tue Jun 17 17:59:25 2003
@@ -218,8 +218,8 @@
def getClassName(self, pickle):
unpickler = self._get_unpickler(pickle)
- module, classname, newargs = unpickler.load()
- return "%s.%s" % (module, classname)
+ cls, newargs = unpickler.load()
+ return cls.__name__
def getGhost(self, pickle):
unpickler = self._get_unpickler(pickle)
=== ZODB4/src/zodb/dbdump.py 1.3 => 1.3.44.1 ===
--- ZODB4/src/zodb/dbdump.py:1.3 Mon Dec 30 16:40:30 2002
+++ ZODB4/src/zodb/dbdump.py Tue Jun 17 17:59:25 2003
@@ -39,11 +39,13 @@
for rec in trans:
if rec.data is None:
fullclass = "undo or abort of object creation"
+ size = 0
else:
# Any object reader will do
reader = SimpleObjectReader()
fullclass = reader.getClassName(rec.data)
dig = md5.new(rec.data).hexdigest()
+ size = len(rec.data)
# special case for testing purposes
if fullclass == "zodb.tests.minpo.MinPO":
obj = zodb_unpickle(rec.data)
@@ -52,8 +54,8 @@
version = "version=%s " % rec.version
else:
version = ''
- print >> outp, " data #%05d oid=%016x %sclass=%s" % \
- (j, u64(rec.oid), version, fullclass)
+ print >> outp, " data #%05d oid=%016x %sclass=%s size=%d" % \
+ (j, u64(rec.oid), version, fullclass, size)
j += 1
print >> outp
i += 1
=== ZODB4/src/zodb/connection.py 1.34 => 1.34.2.1 ===
--- ZODB4/src/zodb/connection.py:1.34 Fri Jun 6 11:24:19 2003
+++ ZODB4/src/zodb/connection.py Tue Jun 17 17:59:25 2003
@@ -41,6 +41,7 @@
import struct
import tempfile
import threading
+from types import StringType
from zope.interface import implements
@@ -396,51 +397,47 @@
# Now is a good time to collect some garbage
self._cache.shrink()
- def _handle_serial(self, store_return, oid=None, change=True):
+ def _handle_serial(self, store_return, oid=None, change=1):
"""Handle the returns from store() and tpc_vote() calls."""
- # XXX We could simplify the storage interface if ZEO would
- # raise the exception itself rather than passing it into the
- # connection.
-
# These calls can return different types depending on whether
# ZEO is used. ZEO uses asynchronous returns that may be
# returned in batches by the ClientStorage. ZEO1 can also
# return an exception object and expect that the Connection
# will raise the exception.
- # When _commit_sub() exceutes a store, there is no need to
+ # When commit_sub() exceutes a store, there is no need to
# update the _p_changed flag, because the subtransaction
- # tpcVote() calls already did this. The change=1 argument
- # exists to allow _commit_sub() to avoid setting the flag
+ # tpc_vote() calls already did this. The change=1 argument
+ # exists to allow commit_sub() to avoid setting the flag
# again.
+
+ # When conflict resolution occurs, the object state held by
+ # the connection does not match what is written to the
+ # database. Invalidate the object here to guarantee that
+ # the new state is read the next time the object is used.
+
if not store_return:
return
- if isinstance(store_return, str):
+ if isinstance(store_return, StringType):
assert oid is not None
- serial = store_return
- obj = self._cache.get(oid)
- if obj is None:
- return
- if serial == ResolvedSerial:
- obj._p_deactivate()
- else:
- if change:
- obj._p_changed = 0
- obj._p_serial = serial
+ self._handle_one_serial(oid, store_return, change)
else:
for oid, serial in store_return:
- if not isinstance(serial, str):
- raise serial
- obj = self._cache.get(oid)
- if obj is None:
- continue
- if serial == ResolvedSerial:
- obj._p_deactivate()
- else:
- if change:
- obj._p_changed = 0
- obj._p_serial = serial
+ self._handle_one_serial(oid, serial, change)
+
+ def _handle_one_serial(self, oid, serial, change=1):
+ if not isinstance(serial, StringType):
+ raise serial
+ obj = self._cache.get(oid, None)
+ if obj is None:
+ return
+ if serial == ResolvedSerial:
+ del obj._p_changed # transition from changed to ghost
+ else:
+ if change:
+ obj._p_changed = 0 # trans. from changed to uptodate
+ obj._p_serial = serial
def _objcommit(self, obj, transaction):
oid = obj._p_oid
=== ZODB4/src/zodb/config.py 1.3 => 1.3.20.1 ===
--- ZODB4/src/zodb/config.py:1.3 Wed Apr 9 17:15:16 2003
+++ ZODB4/src/zodb/config.py Tue Jun 17 17:59:25 2003
@@ -11,182 +11,162 @@
# FOR A PARTICULAR PURPOSE
#
##############################################################################
-"""Default storage types.
+"""Open database and storage from a configuration.
-Adapted from DBTab/StorageTypes.py.
-"""
+$Id$"""
-import re
+import os
+import StringIO
-from ZConfig.Config import asBoolean
+import ZConfig
+import zodb.db
-def convertFileStorageArgs(quota=None, stop=None, **kw):
- if kw.has_key('name'):
- # FileStorage doesn't accept a 'name' arg
- del kw['name']
- if quota is not None:
- kw['quota'] = long(quota) or None
- if stop is not None:
- stop = long(stop)
- if not stop:
- stop = None
- else:
- from zodb.utils import p64
- stop = p64(stop)
- kw['stop'] = stop
-
- # Boolean args
- for name in (
- 'create', 'read_only'
- ):
- if kw.has_key(name):
- kw[name] = asBoolean(kw[name])
-
- return kw
-
-
-# Match URLs of the form 'zeo://zope.example.com:1234'
-zeo_url_re = re.compile('zeo:/*(?P<host>[A-Za-z0-9\.-]+):(?P<port>[0-9]+)')
-
-def convertAddresses(s):
- # Allow multiple addresses using semicolons as a split character.
- res = []
- for a in s.split(';'):
- a = a.strip()
- if a:
- mo = zeo_url_re.match(a)
- if mo is not None:
- # ZEO URL
- host, port = mo.groups()
- res.append((host, int(port)))
- else:
- # Socket file
- res.append(a)
- return res
-
-
-def convertClientStorageArgs(addr=None, **kw):
- if addr is None:
- raise RuntimeError, 'An addr parameter is required for ClientStorage.'
- kw['addr'] = convertAddresses(addr)
-
- # Integer args
- for name in (
- 'cache_size', 'min_disconnect_poll', 'max_disconnect_poll',
- ):
- if kw.has_key(name):
- kw[name] = int(kw[name])
-
- # Boolean args
- for name in (
- 'wait', 'read_only', 'read_only_fallback',
- ):
- if kw.has_key(name):
- kw[name] = asBoolean(kw[name])
-
- # The 'client' parameter must be None to be false. Yuck.
- if kw.has_key('client') and not kw['client']:
- kw['client'] = None
-
- return kw
-
-
-# Currently unused
-def convertBDBStorageArgs(**kw):
- from zodb.storage.base import BerkeleyConfig
- config = BerkeleyConfig()
- for name in dir(BerkeleyConfig):
- if name.startswith('_'):
- continue
- val = kw.get(name)
- if val is not None:
- if name == 'read_only':
- val = asBoolean(val)
- elif name != 'logdir':
- val = int(val)
- setattr(config, name, val)
- del kw[name]
- # XXX: Nobody ever passes in env
- assert not kw.has_key('env')
- kw['config'] = config
- return kw
-
-
-storage_types = {
- # A mapping from "type" (i.e. class) to 2-tuple of (module, converter).
- # converter may be None for no conversion necessary. type and module are
- # both strings, and module should be the dotted path for use in an
- # __import__().
- 'FileStorage' : ('zodb.storage.file', convertFileStorageArgs),
- 'MappingStorage' : ('zodb.storage.mapping', None),
- 'ClientStorage' : ('zodb.zeo.client', convertClientStorageArgs),
- 'BDBFullStorage' : ('zodb.storage.bdbfull', convertBDBStorageArgs),
- 'BDBMinimalStorage' : ('zodb.storage.bdbminimal', convertBDBStorageArgs),
- 'MemoryFullStorage' : ('zodb.storage.memory', convertBDBStorageArgs),
- 'MemoryMinimalStorage': ('zodb.storage.memory', convertBDBStorageArgs),
- }
-
-
-"""Higher-level support for configuring storages.
-
-Storages are configured a la DBTab.
-
-A storage section has the form
-
- <Storage Name (dependent)>
- # For example
- type FileStorage
- file_name var/Data.fs
- read_only 1
- </Storage>
-
-where Name and (dependent) are optional. Once you have retrieved the
-section object (probably with getSection("Storage", name), the
-function creatStorage() in this module will create the storage object
-for you.
-"""
-
-
-
-def createStorage(section):
- """Create a storage specified by a configuration section."""
- klass, args = getStorageInfo(section)
- return klass(**args)
-
-def getStorageInfo(section):
- """Extract a storage description from a configuration section.
-
- Return a tuple (klass, args) where klass is the storage class and
- args is a dictionary of keyword arguments. To create the storage,
- call klass(**args).
+db_schema_path = os.path.join(zodb.__path__[0], "config.xml")
+_db_schema = None
+
+s_schema_path = os.path.join(zodb.__path__[0], "storage.xml")
+_s_schema = None
+
+def getDbSchema():
+ global _db_schema
+ if _db_schema is None:
+ _db_schema = ZConfig.loadSchema(db_schema_path)
+ return _db_schema
+
+def getStorageSchema():
+ global _s_schema
+ if _s_schema is None:
+ _s_schema = ZConfig.loadSchema(s_schema_path)
+ return _s_schema
+
+def databaseFromString(s):
+ return databaseFromFile(StringIO.StringIO(s))
+
+def databaseFromFile(f):
+ config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
+ return databaseFromConfig(config.database)
+
+def databaseFromURL(url):
+ config, handler = ZConfig.loadConfig(getDbSchema(), url)
+ return databaseFromConfig(config.database)
+
+def databaseFromConfig(section):
+ return section.open()
+
+def storageFromString(s):
+ return storageFromFile(StringIO.StringIO(s))
+
+def storageFromFile(f):
+ config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
+ return storageFromConfig(config.storage)
+
+def storageFromURL(url):
+ config, handler = ZConfig.loadConfig(getStorageSchema(), url)
+ return storageFromConfig(config.storage)
+
+def storageFromConfig(section):
+ return section.open()
+
+
+class BaseConfig:
+ """Object representing a configured storage or database.
+
+ Methods:
+
+ open() -- open and return the configured object
+
+ Attributes:
+
+ name -- name of the storage
- Adapted from DatabaseFactory.setStorageParams() in DBTab.py.
"""
- type = section.get("type")
- if not type:
- raise RuntimeError, "A storage type is required"
- module = None
- pos = type.rfind(".")
- if pos >= 0:
- # Specified the module
- module, type = type[:pos], type[pos+1:]
- converter = None
- if not module:
- # Use a default module and argument converter.
- info = storage_types.get(type)
- if not info:
- raise RuntimeError, "Unknown storage type: %s" % type
- module, converter = info
- m = __import__(module, {}, {}, [type])
- klass = getattr(m, type)
-
- args = {}
- if section.name:
- args["name"] = section.name
- for key in section.keys():
- if key.lower() != "type":
- args[key] = section.get(key)
- if converter is not None:
- args = converter(**args)
- return (klass, args)
+
+ def __init__(self, config):
+ self.config = config
+ self.name = config.getSectionName()
+
+ def open(self):
+ """Open and return the storage object."""
+ raise NotImplementedError
+
+class ZODBDatabase(BaseConfig):
+
+ def open(self):
+ section = self.config
+ return zodb.db.DB(section.storage.open(),
+ pool_size=section.pool_size,
+ cache_size=section.cache_size,
+ version_pool_size=section.version_pool_size,
+ version_cache_size=section.version_cache_size)
+
+class MappingStorage(BaseConfig):
+
+ def open(self):
+ from zodb.storage.mapping import MappingStorage
+ return MappingStorage(self.config.name)
+
+class DemoStorage(BaseConfig):
+
+ def open(self):
+ from zodb.storage.demo import DemoStorage
+ if self.config.base:
+ base = self.config.base.open()
+ else:
+ base = None
+ return DemoStorage(self.config.name,
+ base=base,
+ quota=self.config.quota)
+
+class FileStorage(BaseConfig):
+
+ def open(self):
+ from zodb.storage.file import FileStorage
+ return FileStorage(self.config.path,
+ create=self.config.create,
+ read_only=self.config.read_only,
+ quota=self.config.quota)
+
+class ZEOClient(BaseConfig):
+
+ def open(self):
+ from zodb.zeo.client import ClientStorage
+ # config.server is a multikey of socket-address values
+ # where the value is a socket family, address tuple.
+ L = [server.address for server in self.config.server]
+ return ClientStorage(
+ L,
+ storage=self.config.storage,
+ cache_size=self.config.cache_size,
+ name=self.config.name,
+ client=self.config.client,
+ var=self.config.var,
+ min_disconnect_poll=self.config.min_disconnect_poll,
+ max_disconnect_poll=self.config.max_disconnect_poll,
+ wait=self.config.wait,
+ read_only=self.config.read_only,
+ read_only_fallback=self.config.read_only_fallback)
+
+class BDBStorage(BaseConfig):
+
+ def open(self):
+ from zodb.storage.base import BerkeleyConfig
+ storageclass = self.get_storageclass()
+ bconf = BerkeleyConfig()
+ for name in dir(BerkeleyConfig):
+ if name.startswith('_'):
+ continue
+ setattr(bconf, name, getattr(self.config, name))
+ return storageclass(self.config.name, config=bconf)
+
+class BDBMinimalStorage(BDBStorage):
+
+ def get_storageclass(self):
+ from zodb.storage.bdbminimal import BDBMinimalStorage
+ return BDBMinimalStorage
+
+class BDBFullStorage(BDBStorage):
+
+ def get_storageclass(self):
+ from zodb.storage.bdbfull import BDBFullStorage
+ return BDBFullStorage