[Zodb-checkins] CVS: Zope3/src/zodb - conflict.py:1.6.10.2 connection.py:1.7.4.3 db.py:1.7.4.2 interfaces.py:1.8.4.2 lockfile.py:1.3.4.1 serialize.py:1.8.4.2 utils.py:1.2.12.2 winlock.c:1.2.12.1
Jeremy Hylton
jeremy@zope.com
Wed, 12 Mar 2003 16:33:12 -0500
Update of /cvs-repository/Zope3/src/zodb
In directory cvs.zope.org:/tmp/cvs-serv25913
Modified Files:
Tag: opaque-pickles-branch
conflict.py connection.py db.py interfaces.py lockfile.py
serialize.py utils.py winlock.c
Log Message:
Update from trunk.
=== Zope3/src/zodb/conflict.py 1.6.10.1 => 1.6.10.2 ===
--- Zope3/src/zodb/conflict.py:1.6.10.1 Mon Feb 10 17:50:12 2003
+++ Zope3/src/zodb/conflict.py Wed Mar 12 16:32:58 2003
@@ -23,7 +23,7 @@
import logging
from zodb.interfaces import ConflictError
-from zodb.serialize import BaseObjectReader, ObjectWriter, getClassMetadata
+from zodb.serialize import BaseObjectReader, ObjectWriter
ResolvedSerial = "rs"
@@ -43,6 +43,7 @@
def __init__(self, ghost, state):
self._class = ghost.__class__
self._state = state
+ self._p_state = 0 # required to make getClassMetadata() happy
def __getattribute__(self, name):
if name == "__class__":
@@ -94,8 +95,13 @@
The exact rules are implementation dependent. This method was
written to make testing easier.
"""
- meta = getClassMetadata(klass=klass)
- return meta in cls.bad_classes
+ # In a ZEO environment, this method isn't as useful. If the
+ # method is called from a client, it will always return False,
+ # because the conflict resolution code runs on the server.
+
+ # This method depends on the representation of class metadata
+ # that is otherwise only used inside zodb.serialize.
+ return (klass, None) in cls.bad_classes
unresolvable = classmethod(unresolvable)
=== Zope3/src/zodb/connection.py 1.7.4.2 => 1.7.4.3 === (574/674 lines abridged)
--- Zope3/src/zodb/connection.py:1.7.4.2 Tue Mar 11 15:17:22 2003
+++ Zope3/src/zodb/connection.py Wed Mar 12 16:32:58 2003
@@ -26,19 +26,13 @@
associated with the Connection that loaded them. When the objects are
modified, the Connection is registered with the current transaction.
-XXX multi-threaded issues
+Synchronization
-One or more application threads could call methods on a Connection or
-interact with it indirectly via objects that the Connection loaded.
-The intent is for a single thread to use a Connection, but this is not
-enforced.
-
-Not sure if its possible for other threads to interact with Connection
-in other ways.
-
-XXX undocumented issues
-
-The Connection supports callbacks on close and commit.
+A Connection instance is not thread-safe. It is designed to support a
+thread model where each thread has its own transaction. If an
+application has more than one thread that uses the connection or the
+transaction the connection is registered with, the application should
+provide locking.
$Id$
"""
@@ -46,7 +40,7 @@
import cPickle
from cStringIO import StringIO
import logging
-import sys
+import tempfile
import threading
import time
from types import StringType, ClassType, TupleType
@@ -54,13 +48,13 @@
from zodb import interfaces
from zodb.conflict import ResolvedSerial
from zodb.export import ExportImport
-from zodb.interfaces import IConnection, ConflictError, IAppConnection, ZERO
-from zodb.serialize import ConnectionObjectReader, ObjectWriter, findrefs
-from zodb.utils import p64, u64, Set
-from zodb.storage.tmpstore import TmpStore
+from zodb.interfaces import *
+from zodb.serialize import ConnectionObjectReader, ObjectWriter
+from zodb.storage.interfaces import IStorage
+from zodb.utils import p64, u64, Set, z64
[-=- -=- -=- 574 lines omitted -=- -=- -=-]
+ self._pos = self._tpos
+
+ def tpcVote(self, transaction):
+ pass
+
+ def tpcFinish(self, transaction, f=None):
+ if transaction is not self._transaction:
+ return
+ if f is not None:
+ f()
+ undo = UndoInfo(self, self._tpos, self._index.copy())
+ self._index.update(self._tindex)
+ self._tindex.clear()
+ self._tpos = self._pos
+ return undo
+
+ def undoLog(self, first, last, filter=None):
+ return ()
+
+ def versionEmpty(self, version):
+ # XXX what is this supposed to do?
+ if version == self._bver:
+ return len(self._index)
+
+ def rollback(self, pos, index):
+ if not (pos <= self._tpos <= self._pos):
+ msg = "transaction rolled back to early point"
+ raise interfaces.RollbackError(msg)
+ self._tpos = self._pos = pos
+ self._index = index
+ self._tindex.clear()
+
+class UndoInfo:
+ """A helper class for rollback.
+
+ The class stores the state necessary for rolling back to a
+ particular time.
+ """
+
+ def __init__(self, store, pos, index):
+ self._store = store
+ self._pos = pos
+ self._index = index
+
+ def current(self, cur_store):
+ """Return true if the UndoInfo is for cur_store."""
+ return self._store is cur_store
+
+ def rollback(self):
+ self._store.rollback(self._pos, self._index)
=== Zope3/src/zodb/db.py 1.7.4.1 => 1.7.4.2 ===
--- Zope3/src/zodb/db.py:1.7.4.1 Mon Feb 10 17:48:18 2003
+++ Zope3/src/zodb/db.py Wed Mar 12 16:32:58 2003
@@ -18,7 +18,7 @@
__metaclass__ = type
-import cPickle, cStringIO, sys
+import sys
from threading import Lock
from time import time, ctime
from types import StringType
@@ -29,6 +29,7 @@
from zodb.serialize import getDBRoot
from zodb.ztransaction import Transaction
from zodb.interfaces import ZERO
+from zodb.utils import z64, Set
from transaction import get_transaction
from transaction.interfaces import IDataManager
@@ -55,10 +56,12 @@
self.log = logging.getLogger("zodb")
- # Allocate locks:
- l=Lock()
- self._a=l.acquire
- self._r=l.release
+ # The lock protects access to the pool data structures.
+ # Store the lock acquire and release methods as methods
+ # of the instance.
+ l = Lock()
+ self._a = l.acquire
+ self._r = l.release
# Setup connection pools and cache info
# _pool is currently available (closed) connections
@@ -148,17 +151,7 @@
def getPoolSize(self):
return self._pool_size
- def begin_invalidation(self):
- # Must be called before first call to invalidate and before
- # the storage lock is held.
- self._a()
-
- def finish_invalidation(self):
- # Must be called after begin_invalidation() and after final
- # invalidate() call.
- self._r()
-
- def invalidate(self, oid, connection=None, version=''):
+ def invalidate(self, oids, connection=None, version=''):
"""Invalidate references to a given oid.
This is used to indicate that one of the connections has committed a
@@ -166,7 +159,6 @@
passed in to prevent useless (but harmless) messages to the
connection.
"""
- assert oid is not None
if connection is not None:
assert version == connection._version
version = connection._version
@@ -174,18 +166,18 @@
# Notify connections
for cc in self._allocated:
if cc is not connection:
- self.invalidateConnection(cc, oid, version)
+ self.invalidateConnection(cc, oids, version)
if self._temps:
# t accumulates all the connections that aren't closed.
t = []
for cc in self._temps:
if cc is not connection:
- self.invalidateConnection(cc, oid, version,
+ self.invalidateConnection(cc, oids, version,
t.append)
self._temps = t
- def invalidateConnection(self, conn, oid, version, alive=None):
+ def invalidateConnection(self, conn, oids, version, alive=None):
"""Send invalidation message to conn for oid on version.
If the modification occurred on a version, an invalidation is
@@ -205,7 +197,7 @@
if alive is not None:
alive(conn)
if not version or conn.getVersion() == version:
- conn.invalidate(oid)
+ conn.invalidate(oids)
def open(self, version='', transaction=None, temporary=0, force=None,
waitflag=1):
@@ -239,7 +231,8 @@
# This is a temporary connection.
# We won't bother with the pools. This will be
# a one-use connection.
- c = Connection(self, version, cache_size=self._cache_size)
+ c = Connection(self, self._storage, version,
+ cache_size=self._cache_size)
self._temps.append(c)
if transaction is not None:
transaction[id(c)] = c
@@ -259,7 +252,8 @@
# size, then we've never reached the limit.
# Allocate a connection and return without
# touching the lock.
- c = Connection(self, version, cache_size=self._cache_size)
+ c = Connection(self, self._storage, version,
+ cache_size=self._cache_size)
self._allocated.append(c)
return c
else:
@@ -367,18 +361,16 @@
self._dest = dest
def _prepare(self, txn):
- self._oids = self._storage.commitVersion(self._version, self._dest,
- txn)
+ self._oids = Set(self._storage.commitVersion(self._version, self._dest,
+ txn))
def commit(self, txn):
super(CommitVersion, self).commit(txn)
- for oid in self._oids:
- self._db.invalidate(oid, version=self._dest)
+ self._db.invalidate(self._oids, version=self._dest)
if self._dest:
# the code above just invalidated the dest version.
# now we need to invalidate the source!
- for oid in self._oids:
- self._db.invalidate(oid, version=self._version)
+ self._db.invalidate(self._oids, version=self._version)
class AbortVersion(SimpleDataManager):
"""An object that will see to version abortion."""
@@ -388,12 +380,11 @@
self._version = version
def _prepare(self, txn):
- self._oids = self._storage.abortVersion(self._version, txn)
+ self._oids = Set(self._storage.abortVersion(self._version, txn))
def commit(self, txn):
super(AbortVersion, self).commit(txn)
- for oid in self._oids:
- self._db.invalidate(oid, version=self._version)
+ self._db.invalidate(self._oids, version=self._version)
class TransactionalUndo(SimpleDataManager):
"""An object that will see to transactional undo."""
@@ -403,9 +394,8 @@
self._tid = tid
def _prepare(self, txn):
- self._oids = self._storage.undo(self._tid, txn)
+ self._oids = Set(self._storage.undo(self._tid, txn))
def commit(self, txn):
super(TransactionalUndo, self).commit(txn)
- for oid in self._oids:
- self._db.invalidate(oid)
+ self._db.invalidate(self._oids)
=== Zope3/src/zodb/interfaces.py 1.8.4.1 => 1.8.4.2 ===
--- Zope3/src/zodb/interfaces.py:1.8.4.1 Mon Feb 10 17:45:57 2003
+++ Zope3/src/zodb/interfaces.py Wed Mar 12 16:32:58 2003
@@ -1,6 +1,6 @@
##############################################################################
#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# Copyright (c) 2001 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
@@ -11,7 +11,17 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-"""ZODB-defined exceptions
+"""ZODB database interfaces and exceptions
+
+The Zope Object Database (ZODB) manages persistent objects using
+pickle-based object serialization. The database has a pluggable
+storage backend.
+
+The IAppDatabase, IAppConnection, and ITransaction interfaces describe
+the public APIs of the database.
+
+The IDatabase, IConnection, and ITransactionAttrs interfaces describe
+private APIs used by the implementation.
$Id$
"""
@@ -49,9 +59,6 @@
'ITransactionAttrs',
]
-ZERO = '\0'*8
-MAXTID = '\377'*8
-
def _fmt_oid(oid):
return "%016x" % zodb.utils.u64(oid)
@@ -227,8 +234,53 @@
o A reference to an object in a different database connection.
"""
+class IAppDatabase(Interface):
+ """Interface exported by database to applications.
+
+ The database contains a graph of objects reachable from the
+ distinguished root object. The root object is a mapping object
+ that can contain arbitrary application data.
+
+ There is only rudimentary support for using more than one database
+ in a single application. The persistent state of an object in one
+ database can not contain a direct reference to an object in
+ another database.
+ """
+
+ def open(version="", transaction=None, temporary=False, force=False,
+ waitflag=True):
+ # XXX Most of these arguments should eventually go away
+ """Open a new database connection."""
+
+ def abortVersion(version):
+ """Abort the locked database version named version."""
+
+ def commitVersion(source, dest=""):
+ """Commit changes from locked database version source to dest.
+
+ The default value of dest means commit the changes to the
+ default version.
+ """
+
+ def pack(time):
+ """Pack database to time."""
+
+ def undo(txnid):
+ """Undo changes caused by transaction txnid."""
+
class IAppConnection(Interface):
- """Interface exported by database connection to applications."""
+ """Interface exported by database connection to applications.
+
+ Each database connection provides an independent copy of the
+ persistent object space. ZODB supports multiple threads by
+ providing each thread with a separate connection.
+
+ Connections are synchronized through database commits and explicit
+ sync() calls. Changes to the object space are only made visible
+ when a transaction commits. When a connection commits its
+ changes, they become visible to other connections. Changes made
+ by other connections are also become visible at this time.
+ """
def root():
"""Return the root of the database."""
@@ -238,6 +290,22 @@
If there is a current transaction, it will be aborted.
"""
+
+ def get(oid):
+ """Return object for `oid`.
+
+ The object may be a ghost.
+ """
+
+class IDatabase(Interface):
+ """Interface between the database and its connections."""
+
+ def invalidate(oids, conn=None, version=""):
+ pass
+
+ def _closeConnection(conn):
+ pass
+
class IConnection(Interface):
"""Interface required of Connection by ZODB DB.
=== Zope3/src/zodb/lockfile.py 1.3 => 1.3.4.1 ===
--- Zope3/src/zodb/lockfile.py:1.3 Wed Feb 5 18:28:34 2003
+++ Zope3/src/zodb/lockfile.py Wed Mar 12 16:32:58 2003
@@ -12,44 +12,62 @@
#
##############################################################################
-from zodb.storage.interfaces import StorageSystemError
+import os
+import errno
-# Try to create a function that creates Unix file locks.
try:
import fcntl
+except ImportError:
+ try:
+ from zodb.winlock import LockFile as _LockFile
+ from zodb.winlock import UnlockFile as _UnlockFile
+ except ImportError:
+ import logging
+ def lock_file(file):
+ logging.warn('ZODB: No file-locking support on this platform')
+
+ # Windows
+ def lock_file(file):
+ # Lock just the first byte
+ _LockFile(file.fileno(), 0, 0, 1, 0)
- lock_file_FLAG = fcntl.LOCK_EX | fcntl.LOCK_NB
+ def unlock_file(file):
+ _UnlockFile(file.fileno(), 0, 0, 1, 0)
+else:
+ # Unix
+ _flags = fcntl.LOCK_EX | fcntl.LOCK_NB
def lock_file(file):
- try:
- un = file.fileno()
- except:
- return # don't care if not a real file
+ fcntl.flock(file.fileno(), _flags)
+
+ def unlock_file(file):
+ # File is automatically unlocked on close
+ pass
+
+
+# This is a better interface to use than the lockfile.lock_file() interface.
+# Creating the instance acquires the lock. The file remains open. Calling
+# close both closes and unlocks the lock file.
+class LockFile:
+ def __init__(self, path):
+ self._path = path
try:
- fcntl.flock(un, lock_file_FLAG)
- except:
- raise StorageSystemError, (
- "Could not lock the database file. There must be\n"
- "another process that has opened the file.\n")
+ self._fp = open(path, 'r+')
+ except IOError, e:
+ if e.errno <> errno.ENOENT: raise
+ self._fp = open(path, 'w+')
+ # Acquire the lock and piss on the hydrant
+ lock_file(self._fp)
+ print >> self._fp, os.getpid()
+ self._fp.flush()
+
+ def close(self):
+ if self._fp is not None:
+ unlock_file(self._fp)
+ self._fp.close()
+ os.unlink(self._path)
+ self._fp = None
-except:
- # Try windows-specific code:
- try:
- from zodb.winlock import LockFile
- def lock_file(file):
- try:
- un=file.fileno()
- except:
- return # don't care if not a real file
-
- try:
- LockFile(un, 0, 0, 1, 0) # just lock the first byte, who cares
- except:
- raise StorageSystemError, (
- "Could not lock the database file. There must be\n"
- "another process that has opened the file.\n")
- except:
- import logging
- def lock_file(file):
- logging.warn("FS: No file-locking support on this platform")
+ def __del__(self):
+ self.close()
=== Zope3/src/zodb/serialize.py 1.8.4.1 => 1.8.4.2 ===
--- Zope3/src/zodb/serialize.py:1.8.4.1 Mon Feb 10 17:44:32 2003
+++ Zope3/src/zodb/serialize.py Wed Mar 12 16:32:58 2003
@@ -38,6 +38,12 @@
part of its instances class metadata will be a persistent reference to
the class.
+If a type requires extra arguments to ``__new__``, then instances of
+that type must never be in the ghost state. A ghost is an object with
+no state, but if extra arguments are passed to ``__new__`` then the
+object has state as soon as it is constructed. If a ghost object is
+found, it is assumed that ``__getnewargs__`` would return None.
+
Persistent references
---------------------
@@ -53,6 +59,8 @@
changed the class of an object, a new record with new class metadata
would be written but all the old references would still include the
old class.
+
+$Id$
"""
__metaclass__ = type
@@ -64,14 +72,14 @@
from zodb.interfaces import ZERO
-def getClassMetadata(obj=None, klass=None):
- if klass is None:
- klass = obj.__class__
- # XXX Not sure I understand the obj==None case
- newargs = None
- if obj is not None and hasattr(obj, "__getnewargs__"):
- newargs = obj.__getnewargs__()
- return klass, newargs
+def getClassMetadata(obj):
+ if obj._p_state == 3:
+ newargs = None
+ else:
+ newargs = getattr(obj, "__getnewargs__", None)
+ if newargs is not None:
+ newargs = newargs()
+ return obj.__class__, newargs
class RootJar:
def newObjectId(self):
@@ -85,6 +93,8 @@
root = PersistentDict()
return writer.getState(root)
+_marker = object()
+
class ObjectWriter:
def __init__(self, jar=None):
@@ -107,11 +117,10 @@
If it is persistent, it returns the oid and sometimes a tuple
with other stuff.
"""
- if not hasattr(obj, '_p_oid'):
+ oid = getattr(obj, "_p_oid", _marker)
+ if oid is _marker:
return None
- oid = obj._p_oid
-
# I'd like to write something like this --
# if isinstance(oid, types.MemberDescriptor):
# -- but I can't because the type doesn't have a canonical name.
@@ -136,9 +145,7 @@
return NewObjectIterator(self._stack)
def getState(self, obj):
- d = obj.__getstate__()
data = self._dump(getClassMetadata(obj), obj.__getstate__())
- # XXX we should calculate oids while we're pickling the data
refs = findrefs(data)
return data, refs
=== Zope3/src/zodb/utils.py 1.2.12.1 => 1.2.12.2 ===
--- Zope3/src/zodb/utils.py:1.2.12.1 Mon Feb 10 17:45:09 2003
+++ Zope3/src/zodb/utils.py Wed Mar 12 16:32:58 2003
@@ -43,10 +43,21 @@
# This must be Python 2.2, which doesn't have a standard sets module.
# ZODB needs only a very limited subset of the Set API.
class Set(dict):
+ def __init__(self, arg=None):
+ if arg:
+ if isinstance(arg, dict):
+ self.update(arg)
+ else:
+ # XXX the proper sets version is much more robust
+ for o in arg:
+ self[o] = 1
def add(self, o):
self[o] = 1
+ def remove(self, o):
+ del self[o]
def __ior__(self, other):
if not isinstance(other, Set):
return NotImplemented
self.update(other)
return self
+
=== Zope3/src/zodb/winlock.c 1.2 => 1.2.12.1 ===
--- Zope3/src/zodb/winlock.c:1.2 Wed Dec 25 09:12:16 2002
+++ Zope3/src/zodb/winlock.c Wed Mar 12 16:32:58 2003
@@ -2,16 +2,16 @@
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
-
+
This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
-
+
****************************************************************************/
-static char winlock_doc_string[] =
+static char winlock_doc_string[] =
"Lock files on Windows."
"\n"
"$Id$\n";
@@ -25,32 +25,55 @@
#include <windows.h>
#include <io.h>
-static PyObject *
+/* LOCK_FUNC is the shared type of Win32 LockFile and UnlockFile. */
+typedef WINBASEAPI BOOL WINAPI LOCK_FUNC(HANDLE, DWORD, DWORD, DWORD, DWORD);
+
+static PyObject *
+common(LOCK_FUNC func, PyObject *args)
+{
+ int fileno;
+ long h, ofslo, ofshi, lenlo, lenhi;
+
+ if (! PyArg_ParseTuple(args, "illll", &fileno,
+ &ofslo, &ofshi,
+ &lenlo, &lenhi))
+ return NULL;
+
+ h = _get_osfhandle(fileno);
+ if (h == -1) {
+ PyErr_SetString(Error, "_get_osfhandle failed");
+ return NULL;
+ }
+ if (func((HANDLE)h, ofslo, ofshi, lenlo, lenhi)) {
+ Py_INCREF(Py_None);
+ return Py_None;
+ }
+ PyErr_SetObject(Error, PyInt_FromLong(GetLastError()));
+ return NULL;
+}
+
+static PyObject *
winlock(PyObject *ignored, PyObject *args)
{
- int fileno;
- long h, ol, oh, ll, lh;
-
- if (! PyArg_ParseTuple(args, "illll", &fileno, &ol, &oh, &ll, &lh))
- return NULL;
-
- if ((h=_get_osfhandle(fileno))==-1) {
- PyErr_SetString(Error, "_get_osfhandle failed");
- return NULL;
- }
- if (LockFile((HANDLE)h, ol, oh, ll, lh)) {
- Py_INCREF(Py_None);
- return Py_None;
- }
- PyErr_SetObject(Error, PyInt_FromLong(GetLastError()));
- return NULL;
+ return common(LockFile, args);
+}
+
+static PyObject *
+winunlock(PyObject *ignored, PyObject *args)
+{
+ return common(UnlockFile, args);
}
static struct PyMethodDef methods[] = {
- {"LockFile", (PyCFunction)winlock, 1,
- "LockFile(fileno, offsetLow, offsetHigh, lengthLow, lengthHigh ) -- "
- "Lock the file associated with fileno"},
- {NULL, NULL} /* sentinel */
+ {"LockFile", (PyCFunction)winlock, METH_VARARGS,
+ "LockFile(fileno, offsetLow, offsetHigh, lengthLow, lengthHigh) -- "
+ "Lock the file associated with fileno"},
+
+ {"UnlockFile", (PyCFunction)winunlock, METH_VARARGS,
+ "UnlockFile(fileno, offsetLow, offsetHigh, lengthLow, lengthHigh) -- "
+ "Unlock the file associated with fileno"},
+
+ {NULL, NULL} /* sentinel */
};
#else
@@ -66,16 +89,17 @@
#define DL_EXPORT(RTYPE) RTYPE
#endif
DL_EXPORT(void)
-initwinlock(void) {
- PyObject *m, *d;
+initwinlock(void)
+{
+ PyObject *m, *d;
- if (!(Error=PyString_FromString("winlock.error")))
- return;
+ if (!(Error=PyString_FromString("winlock.error")))
+ return;
- /* Create the module and add the functions */
- m = Py_InitModule4("winlock", methods, winlock_doc_string, (PyObject*)NULL,
- PYTHON_API_VERSION);
+ /* Create the module and add the functions */
+ m = Py_InitModule4("winlock", methods, winlock_doc_string,
+ (PyObject*)NULL, PYTHON_API_VERSION);
- d = PyModule_GetDict(m);
- PyDict_SetItemString(d, "error", Error);
+ d = PyModule_GetDict(m);
+ PyDict_SetItemString(d, "error", Error);
}