Tue, 18 Dec 2001 12:04:20 -0500

Update of /cvs-repository/StandaloneZODB/ZODB
In directory cvs.zope.org:/tmp/cvs-serv28929/ZODB

Modified Files:
      Tag: Standby-branch
	BaseStorage.py Connection.py DB.py DemoStorage.py 
	ExportImport.py FileStorage.py MappingStorage.py Mount.py 
	POSException.py PersistentMapping.py TimeStamp.c 
	Transaction.py UndoLogCompatible.py ZApplication.py 
	bpthread.py cPersistence.c cPickleCache.c coptimizations.c 
	dbmStorage.py winlock.c 
Added Files:
      Tag: Standby-branch
	fsIndex.py 
Removed Files:
      Tag: Standby-branch
	Setup 
Log Message:
Merge StandaloneZODB-1_0-branch into Standby-branch


=== Added File StandaloneZODB/ZODB/fsIndex.py ===
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
# 
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
# 
##############################################################################
"""Implement an OID to File-position (long integer) mapping
"""
# 
# To save space, we do two things:
# 
#     1. We split the keys (OIDS) into 6-byte prefixes and 2-byte suffixes.
#        We use the prefixes as keys in a mapping from prefix to mappings
#        of suffix to data:
# 
#           data is  {prefix -> {suffix -> data}}
# 
#     2. We limit the data size to 48 bits. This should allow databases
#        as large as 256 terabytes.
# 
# Mostof the space is consumed by items in the mappings from 2-byte
# suffix to 6-byte data. This should reduce the overall memory usage to
# 8-16 bytes per OID.
# 
# We use p64 to convert integers to 8-byte strings and lop off the two
# high-order bytes when saving. On loading data, we add the leading
# bytes back before using U64 to convert the data back to (long)
# integers.

from BTrees._fsBTree import fsBTree as _fsBTree

import struct

# convert between numbers and six-byte strings

_t32 = 1L<< 32

def num2str(n):
    h, l = divmod(long(n), _t32)
    return struct.pack(">HI", h, l)

def str2num(s):
    h, l = struct.unpack(">HI", s)
    if h:
        return (long(h) << 32) + l
    else:
       return l

class fsIndex:

    def __init__(self):
        self._data = {}

    def __getitem__(self, key):
        return str2num(self._data[key[:6]][key[6:]])

    def get(self, key, default=None):
        tree = self._data.get(key[:6], default)
        if tree is default:
            return default
        v = tree.get(key[6:], default)
        if v is default:
            return default
        return str2num(v)

    def __setitem__(self, key, value):
        value = num2str(value)
        treekey = key[:6]
        tree = self._data.get(treekey)
        if tree is None:
            tree = _fsBTree()
            self._data[treekey] = tree
        tree[key[6:]] = value

    def __len__(self):
        r = 0
        for tree in self._data.values():
            r += len(tree)
        return r

    def update(self, mapping):
        for k, v in mapping.items():
            self[k] = v

    def has_key(self, key):
        v=self.get(key, self)
        return v is not self

    def clear(self):
        self._data.clear()

    def keys(self):
        r = []
        for prefix, tree in self._data.items():
            for suffix in tree.keys():
                r.append(prefix + suffix)
        return r

    def items(self):
        r = []
        for prefix, tree in self._data.items():
            for suffix, v in tree.items():
                r.append(((prefix + suffix), str2num(v)))
        return r

    def values(self):
        r = []
        for prefix, tree in self._data.items():
            for v in tree.values():
                r.append(str2num(v))
        return r


=== StandaloneZODB/ZODB/BaseStorage.py 1.16 => 1.16.8.1 ===


=== StandaloneZODB/ZODB/Connection.py 1.61 => 1.61.8.1 ===
 
 from cPickleCache import PickleCache
-from POSException import ConflictError
+from POSException import ConflictError, ReadConflictError
 from ExtensionClass import Base
 import ExportImport, TmpStore
 from zLOG import LOG, ERROR, BLATHER
@@ -248,7 +248,7 @@
                 or
                 invalid(None)
                 ):
-                raise ConflictError, `oid`
+                raise ConflictError(object=object)
             self._invalidating.append(oid)
 
         else:
@@ -315,7 +315,7 @@
                     or
                     invalid(None)
                     ):
-                    raise ConflictError, `oid`
+                    raise ConflictError(object=object)
                 self._invalidating.append(oid)
                 
             klass = object.__class__
@@ -459,7 +459,7 @@
             if invalid(oid) or invalid(None):
                 if not hasattr(object.__class__, '_p_independent'):
                     get_transaction().register(self)
-                    raise ConflictError(`oid`, `object.__class__`)
+                    raise ReadConflictError(object=object)
                 invalid=1
             else:
                 invalid=0
@@ -484,7 +484,7 @@
                     except KeyError: pass
                 else:
                     get_transaction().register(self)
-                    raise ConflictError(`oid`, `object.__class__`)
+                    raise ConflictError(object=object)
 
         except ConflictError:
             raise
@@ -544,7 +544,7 @@
 
     def tpc_begin(self, transaction, sub=None):
         if self._invalid(None): # Some nitwit invalidated everything!
-            raise ConflictError, "transaction already invalidated"
+            raise ConflictError("transaction already invalidated")
         self._invalidating=[]
         self._creating=[]
 


=== StandaloneZODB/ZODB/DB.py 1.36 => 1.36.6.1 ===


=== StandaloneZODB/ZODB/DemoStorage.py 1.8 => 1.8.8.1 ===
                     nv=old
 
-                if serial != oserial: raise POSException.ConflictError
+                if serial != oserial:
+                    raise POSException.ConflictError(serials=(oserial, serial))
                 
             serial=self._serial
             r=[oid, serial, old, version and (version, nv) or None, data]


=== StandaloneZODB/ZODB/ExportImport.py 1.12 => 1.12.8.1 ===
 ##############################################################################
 
-"""Support for database export and import.
-"""
+"""Support for database export and import."""
 
 import POSException, string
 


=== StandaloneZODB/ZODB/FileStorage.py 1.76 => 1.76.8.1 ===
 __version__='$Revision$'[11:-2]
 
-import struct, time, os, bpthread, string, base64, sys
+import struct, time, os, string, base64, sys
 from struct import pack, unpack
-from cPickle import loads
 import POSException
 from POSException import UndoError
 from TimeStamp import TimeStamp
@@ -127,9 +126,15 @@
 from zLOG import LOG, WARNING, ERROR, PANIC, register_subsystem
 register_subsystem('ZODB FS')
 import BaseStorage
-from cPickle import Pickler, Unpickler
+from cPickle import Pickler, Unpickler, loads
 import ConflictResolution
 
+try:
+    from fsIndex import fsIndex
+except ImportError:
+    def fsIndex():
+        return {}
+
 try: from posix import fsync
 except: fsync=None
 
@@ -262,7 +267,8 @@
         self._index_get=index.get
         self._vindex_get=vindex.get
 
-    def __len__(self): return len(self._index)
+    def __len__(self):
+        return len(self._index)
 
     def _newIndexes(self):
         # hook to use something other than builtin dict
@@ -632,8 +638,8 @@
                 if serial != oserial:
                     data=self.tryToResolveConflict(oid, oserial, serial, data)
                     if not data:
-                        raise POSException.ConflictError, (
-                            serial, oserial)
+                        raise POSException.ConflictError(oid=oid,
+                                                serials=(oserial, serial))
             else:
                 oserial=serial
                     
@@ -1218,7 +1224,7 @@
     
             rootl=[z64]
             pop=rootl.pop
-            pindex={}
+            pindex=fsIndex()
             referenced=pindex.has_key
             _load=self._load
             _loada=self._loada
@@ -1248,7 +1254,7 @@
     
             # Index for non-version data.  This is a temporary structure
             # to reduce I/O during packing
-            nvindex={}
+            nvindex=fsIndex()
     
             # Cache a bunch of methods
             seek=file.seek


=== StandaloneZODB/ZODB/MappingStorage.py 1.4 => 1.4.8.1 ===
                 old=self._index[oid]
                 oserial=old[:8]
-                if serial != oserial: raise POSException.ConflictError
+                if serial != oserial:
+                    raise POSException.ConflictError(serials=(oserial, serial))
                 
             serial=self._serial
             self._tindex.append((oid,serial+data))


=== StandaloneZODB/ZODB/Mount.py 1.13 => 1.13.4.1 ===


=== StandaloneZODB/ZODB/POSException.py 1.8 => 1.8.8.1 ===
 # 
 ##############################################################################
-'''BoboPOS-defined exceptions
+"""BoboPOS-defined exceptions
 
-$Id$'''
-__version__='$Revision$'[11:-2]
+$Id$"""
+__version__ = '$Revision$'.split()[-2:][0]
 
 from string import join
-StringType=type('')
-DictType=type({})
+from types import StringType, DictType
+from ZODB import utils
 
 class POSError(Exception):
     """Persistent object system error
@@ -28,10 +28,94 @@
     """
 
 class ConflictError(TransactionError):
-    """Two transactions tried to modify the same object at once
+    """Two transactions tried to modify the same object at once.  This
+    transaction should be resubmitted.
 
-    This transaction should be resubmitted.
+    Instance attributes:
+      oid : string
+        the OID (8-byte packed string) of the object in conflict
+      class_name : string
+        the fully-qualified name of that object's class
+      message : string
+        a human-readable explanation of the error
+      serials : (string, string)
+        a pair of 8-byte packed strings; these are the serial numbers
+        (old and new) of the object in conflict.  (Serial numbers are
+        closely related [equal?] to transaction IDs; a ConflictError may
+        be triggered by a serial number mismatch.)
+
+    The caller should pass either object or oid as a keyword argument,
+    but not both of them.  If object is passed, it should be a
+    persistent object with an _p_oid attribute.
     """
+
+    def __init__(self, message=None, object=None, oid=None, serials=None):
+        if message is None:
+            self.message = "database conflict error"
+        else:
+            self.message = message
+
+        if object is None:
+            self.oid = None
+            self.class_name = None
+        else:
+            self.oid = object._p_oid
+            klass = object.__class__
+            self.class_name = klass.__module__ + "." + klass.__name__
+
+        if oid is not None:
+            assert self.oid is None
+            self.oid = oid
+
+        self.serials = serials
+
+    def __str__(self):
+        extras = []
+        if self.oid:
+            extras.append("oid %016x" % utils.U64(self.oid))
+        if self.class_name:
+            extras.append("class %s" % self.class_name)
+        if self.serials:
+            extras.append("serial was %016x, now %016x" %
+                          tuple(map(utils.U64, self.serials)))
+        if extras:
+            return "%s (%s)" % (self.message, ", ".join(extras))
+        else:
+            return self.message
+
+    def get_oid(self):
+        return self.oid
+
+    def get_class_name(self):
+        return self.class_name
+
+    def get_old_serial(self):
+        return self.serials[0]
+
+    def get_new_serial(self):
+        return self.serials[1]
+
+    def get_serials(self):
+        return self.serials
+
+
+class ReadConflictError(ConflictError):
+    """A conflict detected at read time -- attempt to read an object
+    that has changed in another transaction (eg. another thread
+    or process).
+    """
+    def __init__(self, message=None, object=None, serials=None):
+        if message is None:
+            message = "database read conflict error"
+        ConflictError.__init__(self, message=message, object=object,
+                               serials=serials)
+
+class BTreesConflictError(ConflictError):
+    """A special subclass for BTrees conflict errors, which return
+    an undocumented four-tuple."""
+    def __init__(self, *btree_args):
+        ConflictError.__init__(self, message="BTrees conflict error")
+        self.btree = btree_args
 
 class VersionError(POSError):
     """An error in handling versions occurred


=== StandaloneZODB/ZODB/PersistentMapping.py 1.17 => 1.17.4.1 ===


=== StandaloneZODB/ZODB/TimeStamp.c 1.10 => 1.10.8.1 ===


=== StandaloneZODB/ZODB/Transaction.py 1.32 => 1.32.8.1 ===


=== StandaloneZODB/ZODB/UndoLogCompatible.py 1.4 => 1.4.8.1 ===
 # 
 ##############################################################################
-"""Provide backward compatability with storages that have undoLog, but not undoInfo."""
-
+"""backwards compatibility for storages that have undoLog but not undoInfo."""
 
 class UndoLogCompatible:
 


=== StandaloneZODB/ZODB/ZApplication.py 1.9 => 1.9.8.1 ===


=== StandaloneZODB/ZODB/bpthread.py 1.3 => 1.3.8.1 ===


=== StandaloneZODB/ZODB/cPersistence.c 1.47 => 1.47.8.1 ===


=== StandaloneZODB/ZODB/cPickleCache.c 1.37 => 1.37.8.1 ===


=== StandaloneZODB/ZODB/coptimizations.c 1.14 => 1.14.8.1 ===


=== StandaloneZODB/ZODB/dbmStorage.py 1.2 => 1.2.8.1 ===


=== StandaloneZODB/ZODB/winlock.c 1.6 => 1.6.8.1 ===

=== Removed File StandaloneZODB/ZODB/Setup ===



