[Zodb-checkins] CVS: Zope/lib/python/ZODB - config.py:1.2.2.1 config.xml:1.2.2.1 ActivityMonitor.py:1.3.4.5 BaseStorage.py:1.20.4.11 Connection.py:1.72.4.10 DB.py:1.43.4.4 DemoStorage.py:1.12.4.11 FileStorage.py:1.95.4.10 POSException.py:1.12.4.13 TimeStamp.c:1.15.58.7 cPersistence.c:1.62.8.7 cPickleCache.c:1.68.8.4
Chris McDonough
chrism@zope.com
Sat, 4 Jan 2003 13:17:00 -0500
Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv3465
Modified Files:
Tag: chrism-install-branch
ActivityMonitor.py BaseStorage.py Connection.py DB.py
DemoStorage.py FileStorage.py POSException.py TimeStamp.c
cPersistence.c cPickleCache.c
Added Files:
Tag: chrism-install-branch
config.py config.xml
Log Message:
Merge with HEAD.
=== Added File Zope/lib/python/ZODB/config.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Open database and storage from a configuration.
$Id: config.py,v 1.2.2.1 2003/01/04 18:16:21 chrism Exp $"""
import os
import StringIO
import ZConfig
import ZODB
schema_path = os.path.join(ZODB.__path__[0], "config.xml")
_schema = None
def getSchema():
global _schema
if _schema is None:
_schema = ZConfig.loadSchema(schema_path)
return _schema
def databaseFromString(s):
return databaseFromFile(StringIO.StringIO(s))
def databaseFromFile(f):
config, handle = ZConfig.loadConfigFile(getSchema(), f)
return databaseFromConfig(config)
def databaseFromURL(url):
config, handler = ZConfig.loadConfig(getSchema(), url)
return databaseFromConfig(config)
def databaseFromConfig(config):
return ZODB.DB(config.storage.open(),
pool_size=config.pool_size,
cache_size=config.cache_size,
version_pool_size=config.version_pool_size,
version_cache_size=config.version_cache_size)
class StorageConfig:
def __init__(self, config):
self.config = config
def open(self):
raise NotImplementedError
class MappingStorage(StorageConfig):
def open(self):
from ZODB.MappingStorage import MappingStorage
return MappingStorage(self.config.name)
class FileStorage(StorageConfig):
def open(self):
from ZODB.FileStorage import FileStorage
return FileStorage(self.config.path,
create=self.config.create,
read_only=self.config.read_only,
stop=self.config.stop,
quota=self.config.quota)
class ZEOClient(StorageConfig):
def open(self):
from ZEO.ClientStorage import ClientStorage
# config.server is a multikey of socket-address values
# where the value is a socket family, address tuple.
L = [addr for family, addr in self.config.server]
return ClientStorage(
L,
storage=self.config.storage,
cache_size=self.config.cache_size,
name=self.config.name,
client=self.config.client,
var=self.config.var,
min_disconnect_poll=self.config.min_disconnect_poll,
max_disconnect_poll=self.config.max_disconnect_poll,
wait=self.config.wait,
read_only=self.config.read_only,
read_only_fallback=self.config.read_only_fallback)
class BDBStorage(StorageConfig):
def open(self):
from BDBStorage.BerkeleyBase import BerkeleyConfig
from BDBStorage.BDBFullStorage import BDBFullStorage
from BDBStorage.BDBMinimalStorage import BDBMinimalStorage
# Figure out which class we want
sectiontype = self.config.getSectionType()
storageclass = {'fullstorage': BDBFullStorage,
'minimalstorage': BDBMinimalStorage,
}[sectiontype]
bconf = BerkeleyConfig()
for name in dir(BerkeleyConfig):
if name.startswith('_'):
continue
setattr(bconf, name, getattr(self.config, name))
return storageclass(self.config.name, config=bconf)
=== Added File Zope/lib/python/ZODB/config.xml ===
<schema type="database">
<sectiongroup type="storage">
<sectiontype type="filestorage" datatype="ZODB.config.FileStorage">
<key name="path" required="yes"/>
<key name="create" datatype="boolean" default="true"/>
<key name="read_only" datatype="boolean" default="false"/>
<key name="stop"/>
<key name="quota" datatype="integer"/>
</sectiontype>
<sectiontype type="mappingstorage" datatype="ZODB.config.MappingStorage">
<key name="name" default="Mapping Storage"/>
</sectiontype>
<sectiontype type="fullstorage" datatype="ZODB.config.BDBStorage">
<key name="name" required="yes" />
<key name="interval" datatype="time-interval" default="2m" />
<key name="kbyte" datatype="integer" default="0" />
<key name="min" datatype="integer" default="0" />
<key name="logdir" />
<key name="cachesize" datatype="byte-size" default="128MB" />
<key name="frequency" datatype="time-interval" default="0" />
<key name="packtime" datatype="time-interval" default="4h" />
<key name="classicpack" datatype="integer" default="0" />
<key name="read_only" datatype="boolean" default="off"/>
</sectiontype>
<!-- XXX Fred promises to make it so minimal storage is just an
extension of fullstorage -->
<sectiontype type="minimalstorage" datatype="ZODB.config.BDBStorage">
<key name="name" required="yes" />
<key name="interval" datatype="time-interval" default="2m" />
<key name="kbyte" datatype="integer" default="0" />
<key name="min" datatype="integer" default="0" />
<key name="logdir" />
<key name="cachesize" datatype="byte-size" default="128MB" />
<key name="frequency" datatype="time-interval" default="0" />
<key name="packtime" datatype="time-interval" default="4h" />
<key name="classicpack" datatype="integer" default="0" />
<key name="read_only" datatype="boolean" default="off"/>
</sectiontype>
<sectiontype type="zeoclient" datatype="ZODB.config.ZEOClient">
<multikey name="server" datatype="socket-address" required="yes"/>
<key name="storage" default="1"/>
<key name="cache_size" datatype="integer" default="20000000"/>
<key name="name" default=""/>
<key name="client"/>
<key name="var"/>
<key name="min_disconnect_poll" datatype="integer" default="5"/>
<key name="max_disconnect_poll" datatype="integer" default="300"/>
<key name="wait" datatype="boolean" default="on"/>
<key name="read_only" datatype="boolean" default="off"/>
<key name="read_only_fallback" datatype="boolean" default="off"/>
</sectiontype>
<sectiontype type="demostorage">
<!--datatype="ZODB.config.DemoStorage"-->
<key name="name" default="Demo Storage"/>
<section type="storage" name="*" attribute="base"/>
<key name="quota" datatype="integer"/>
</sectiontype>
</sectiongroup>
<!-- the rest is the actual configuration for the database -->
<section type="storage" name="*" attribute="storage"/>
<key name="cache_size" datatype="integer" default="5000"/>
<key name="pool_size" datatype="integer" default="7"/>
<key name="version_pool_size" datatype="integer" default="3"/>
<key name="version_cache_size" datatype="integer" default="100"/>
</schema>
=== Zope/lib/python/ZODB/ActivityMonitor.py 1.3.4.4 => 1.3.4.5 ===
=== Zope/lib/python/ZODB/BaseStorage.py 1.20.4.10 => 1.20.4.11 ===
--- Zope/lib/python/ZODB/BaseStorage.py:1.20.4.10 Fri Jan 3 01:37:19 2003
+++ Zope/lib/python/ZODB/BaseStorage.py Sat Jan 4 13:16:20 2003
@@ -12,11 +12,9 @@
#
##############################################################################
"""Handy standard storage machinery
-"""
-# Do this portably in the face of checking out with -kv
-import string
-__version__ = string.split('$Revision$')[-2:][0]
+$Id$
+"""
import cPickle
import ThreadLock, bpthread
import time, UndoLogCompatible
@@ -277,8 +275,8 @@
restoring = 1
else:
restoring = 0
- for transaction in other.iterator():
-
+ fiter = other.iterator()
+ for transaction in fiter:
tid=transaction.tid
if _ts is None:
_ts=TimeStamp(tid)
@@ -312,6 +310,8 @@
self.tpc_vote(transaction)
self.tpc_finish(transaction)
+
+ fiter.close()
class TransactionRecord:
"""Abstract base class for iterator protocol"""
=== Zope/lib/python/ZODB/Connection.py 1.72.4.9 => 1.72.4.10 ===
=== Zope/lib/python/ZODB/DB.py 1.43.4.3 => 1.43.4.4 ===
=== Zope/lib/python/ZODB/DemoStorage.py 1.12.4.10 => 1.12.4.11 ===
=== Zope/lib/python/ZODB/FileStorage.py 1.95.4.9 => 1.95.4.10 ===
--- Zope/lib/python/ZODB/FileStorage.py:1.95.4.9 Fri Jan 3 01:37:19 2003
+++ Zope/lib/python/ZODB/FileStorage.py Sat Jan 4 13:16:20 2003
@@ -124,7 +124,7 @@
import struct
import sys
import time
-from types import StringType
+from types import StringType, DictType
from struct import pack, unpack
try:
@@ -137,7 +137,12 @@
from ZODB.TimeStamp import TimeStamp
from ZODB.lock_file import lock_file
from ZODB.utils import p64, u64, cp, z64
-from ZODB.fsIndex import fsIndex
+
+try:
+ from ZODB.fsIndex import fsIndex
+except ImportError:
+ def fsIndex():
+ return {}
from zLOG import LOG, BLATHER, WARNING, ERROR, PANIC
@@ -203,6 +208,8 @@
# default pack time is 0
_packt = z64
+ _records_before_save = 10000
+
def __init__(self, file_name, create=0, read_only=0, stop=None,
quota=None):
@@ -270,7 +277,9 @@
r = self._restore_index()
if r is not None:
+ self._used_index = 1 # Marker for testing
index, vindex, start, maxoid, ltid = r
+
self._initIndex(index, vindex, tindex, tvindex)
self._pos, self._oid, tid = read_index(
self._file, file_name, index, vindex, tindex, stop,
@@ -278,10 +287,15 @@
read_only=read_only,
)
else:
+ self._used_index = 0 # Marker for testing
self._pos, self._oid, tid = read_index(
self._file, file_name, index, vindex, tindex, stop,
read_only=read_only,
)
+ self._save_index()
+
+ self._records_before_save = max(self._records_before_save,
+ len(self._index))
self._ltid = tid
# self._pos should always point just past the last
@@ -314,6 +328,7 @@
# hook to use something other than builtin dict
return fsIndex(), {}, {}, {}
+ _saved = 0
def _save_index(self):
"""Write the database index to a file to support quick startup."""
@@ -329,6 +344,7 @@
p.dump(info)
f.flush()
f.close()
+
try:
try:
os.remove(index_name)
@@ -337,6 +353,8 @@
os.rename(tmp_name, index_name)
except: pass
+ self._saved += 1
+
def _clear_index(self):
index_name = self.__name__ + '.index'
if os.path.exists(index_name):
@@ -354,58 +372,77 @@
object positions cause zero to be returned.
"""
- if pos < 100: return 0
- file=self._file
- seek=file.seek
- read=file.read
+ if pos < 100:
+ return 0 # insane
+ file = self._file
+ seek = file.seek
+ read = file.read
seek(0,2)
- if file.tell() < pos: return 0
- ltid=None
+ if file.tell() < pos:
+ return 0 # insane
+ ltid = None
- while 1:
+ max_checked = 5
+ checked = 0
+
+ while checked < max_checked:
seek(pos-8)
- rstl=read(8)
- tl=u64(rstl)
- pos=pos-tl-8
- if pos < 4: return 0
+ rstl = read(8)
+ tl = u64(rstl)
+ pos = pos-tl-8
+ if pos < 4:
+ return 0 # insane
seek(pos)
s = read(TRANS_HDR_LEN)
tid, stl, status, ul, dl, el = unpack(TRANS_HDR, s)
- if not ltid: ltid=tid
- if stl != rstl: return 0 # inconsistent lengths
- if status == 'u': continue # undone trans, search back
- if status not in ' p': return 0
- if tl < (TRANS_HDR_LEN + ul + dl + el): return 0
- tend=pos+tl
- opos=pos+(TRANS_HDR_LEN + ul + dl + el)
- if opos==tend: continue # empty trans
+ if not ltid:
+ ltid = tid
+ if stl != rstl:
+ return 0 # inconsistent lengths
+ if status == 'u':
+ continue # undone trans, search back
+ if status not in ' p':
+ return 0 # insane
+ if tl < (TRANS_HDR_LEN + ul + dl + el):
+ return 0 # insane
+ tend = pos+tl
+ opos = pos+(TRANS_HDR_LEN + ul + dl + el)
+ if opos == tend:
+ continue # empty trans
- while opos < tend:
+ while opos < tend and checked < max_checked:
# Read the data records for this transaction
seek(opos)
- h=read(DATA_HDR_LEN)
- oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
- tloc=u64(stloc)
- plen=u64(splen)
+ h = read(DATA_HDR_LEN)
+ oid, serial, sprev, stloc, vlen, splen = unpack(DATA_HDR, h)
+ tloc = u64(stloc)
+ plen = u64(splen)
+
+ dlen = DATA_HDR_LEN+(plen or 8)
+ if vlen:
+ dlen = dlen+(16+vlen)
- dlen=DATA_HDR_LEN+(plen or 8)
- if vlen: dlen=dlen+(16+vlen)
+ if opos+dlen > tend or tloc != pos:
+ return 0 # insane
- if opos+dlen > tend or tloc != pos: return 0
+ if index.get(oid, 0) != opos:
+ return 0 # insane
- if index.get(oid, 0) != opos: return 0
+ checked += 1
- opos=opos+dlen
+ opos = opos+dlen
return ltid
def _restore_index(self):
"""Load database index to support quick startup."""
- try:
- f = open("%s.index" % self.__name__, 'rb')
- except:
- return None
- p = Unpickler(f)
+ file_name=self.__name__
+ index_name=file_name+'.index'
+
+ try: f=open(index_name,'rb')
+ except: return None
+
+ p=Unpickler(f)
try:
info=p.load()
@@ -422,6 +459,23 @@
return None
pos = long(pos)
+ if isinstance(index, DictType) and not self._is_read_only:
+ # Convert to fsIndex
+ newindex = fsIndex()
+ if type(newindex) is not type(index):
+ # And we have fsIndex
+ newindex.update(index)
+
+ # Now save the index
+ f = open(index_name, 'wb')
+ p = Pickler(f, 1)
+ info['index'] = newindex
+ p.dump(info)
+ f.close()
+
+ # Now call this method again to get the new data
+ return self._restore_index()
+
tid = self._sane(index, pos)
if not tid:
return None
@@ -955,6 +1009,9 @@
finally:
self._lock_release()
+ # Keep track of the number of records that we've written
+ _records_written = 0
+
def _finish(self, tid, u, d, e):
nextpos=self._nextpos
if nextpos:
@@ -967,10 +1024,20 @@
if fsync is not None: fsync(file.fileno())
- self._pos=nextpos
-
+ self._pos = nextpos
+
self._index.update(self._tindex)
self._vindex.update(self._tvindex)
+
+ # Update the number of records that we've written
+ # +1 for the transaction record
+ self._records_written += len(self._tindex) + 1
+ if self._records_written >= self._records_before_save:
+ self._save_index()
+ self._records_written = 0
+ self._records_before_save = max(self._records_before_save,
+ len(self._index))
+
self._ltid = tid
def _abort(self):
@@ -1210,7 +1277,8 @@
while pos < tend:
self._file.seek(pos)
h = self._file.read(DATA_HDR_LEN)
- oid, serial, sprev, stloc, vlen, splen = struct.unpack(DATA_HDR, h)
+ oid, serial, sprev, stloc, vlen, splen = \
+ struct.unpack(DATA_HDR, h)
if failed(oid):
del failures[oid] # second chance!
plen = u64(splen)
@@ -1966,6 +2034,7 @@
id in the data. The transaction id is the tid of the last
transaction.
"""
+
read = file.read
seek = file.seek
seek(0, 2)
@@ -2001,7 +2070,7 @@
if tid <= ltid:
warn("%s time-stamp reduction at %s", name, pos)
- ltid=tid
+ ltid = tid
tl=u64(stl)
@@ -2074,7 +2143,12 @@
if vlen:
dlen=dlen+(16+vlen)
read(16)
+ pv=u64(read(8))
version=read(vlen)
+ # Jim says: "It's just not worth the bother."
+ #if vndexpos(version, 0) != pv:
+ # panic("%s incorrect previous version pointer at %s",
+ # name, pos)
vindex[version]=pos
if pos+dlen > tend or tloc != tpos:
@@ -2223,8 +2297,11 @@
self._stop = stop
def __len__(self):
- # This is a lie. It's here only for Python 2.1 support for
- # list()-ifying these objects.
+ # Define a bogus __len__() to make the iterator work
+ # with code like builtin list() and tuple() in Python 2.1.
+ # There's a lot of C code that expects a sequence to have
+ # an __len__() but can cope with any sort of mistake in its
+ # implementation. So just return 0.
return 0
def close(self):
@@ -2362,7 +2439,6 @@
class RecordIterator(Iterator, BaseStorage.TransactionRecord):
"""Iterate over the transactions in a FileStorage file."""
-
def __init__(self, tid, status, user, desc, ext, pos, tend, file, tpos):
self.tid = tid
self.status = status
=== Zope/lib/python/ZODB/POSException.py 1.12.4.12 => 1.12.4.13 ===
--- Zope/lib/python/ZODB/POSException.py:1.12.4.12 Fri Jan 3 01:37:20 2003
+++ Zope/lib/python/ZODB/POSException.py Sat Jan 4 13:16:20 2003
@@ -219,4 +219,6 @@
o A reference to a wrapped persistent object.
o A reference to an object in a different database connection.
+
+ XXX The exception ought to have a member that is the invalid object.
"""
=== Zope/lib/python/ZODB/TimeStamp.c 1.15.58.6 => 1.15.58.7 ===
=== Zope/lib/python/ZODB/cPersistence.c 1.62.8.6 => 1.62.8.7 ===
=== Zope/lib/python/ZODB/cPickleCache.c 1.68.8.3 => 1.68.8.4 ===