[Zope3-checkins] CVS: Zope3/src/zodb/storage - base.py:1.15 file.py:1.10 interfaces.py:1.7 mapping.py:1.5
Jeremy Hylton
jeremy@zope.com
Tue, 25 Feb 2003 13:55:34 -0500
Update of /cvs-repository/Zope3/src/zodb/storage
In directory cvs.zope.org:/tmp/cvs-serv23205/src/zodb/storage
Modified Files:
base.py file.py interfaces.py mapping.py
Log Message:
Merge the ZODB3-2-integration branch to the trunk.
The primary changes here are to port the many new ZEO features from
ZODB 3.2. There are also many changes to the test suite.
python2.2 tells me: Ran 3755 tests in 560.277s
=== Zope3/src/zodb/storage/base.py 1.14 => 1.15 ===
--- Zope3/src/zodb/storage/base.py:1.14 Tue Feb 11 10:59:27 2003
+++ Zope3/src/zodb/storage/base.py Tue Feb 25 13:55:03 2003
@@ -92,9 +92,6 @@
raise StorageTransactionError(self, transaction)
return []
- def close(self):
- pass
-
def sortKey(self):
# An implementation should override this if it can be shared
# by multiple processes and / or guarantee that self._name is
=== Zope3/src/zodb/storage/file.py 1.9 => 1.10 ===
--- Zope3/src/zodb/storage/file.py:1.9 Tue Feb 11 10:59:27 2003
+++ Zope3/src/zodb/storage/file.py Tue Feb 25 13:55:03 2003
@@ -212,13 +212,6 @@
class FileStorageQuotaError(FileStorageError, StorageSystemError):
"""File storage quota exceeded."""
-def DB(file_name, create=0, read_only=0, stop=None, quota=None,
- pool_size=7, cache_size=400):
- """Create new object database using FileStorage file_name."""
- fs = FileStorage(file_name, create, read_only, stop, quota)
- db = zodb.db.DB(fs, pool_size, cache_size)
- return db
-
class FileStorageFormatter:
"""Mixin class that can read and write the low-level format."""
@@ -1989,171 +1982,6 @@
print "Recovered file, lost %s, ended up with %s bytes" % (
pos-opos, npos)
-
-
-
-def Xread_index(file, name, index, vindex, tindex, stop='\377'*8,
- ltid=z64, start=4L, maxoid=z64, recover=0, read_only=0):
- """Scan the entire file storage and recreate the index.
-
- Returns file position, max oid, and last transaction id. It also
- stores index information in the three dictionary arguments.
-
- Arguments:
- file -- a file object (the Data.fs)
- name -- the name of the file (presumably file.name)
- index -- dictionary, oid -> data record
- vindex -- dictionary, oid -> data record for version data
- tindex -- dictionary, oid -> data record
- XXX tindex is cleared before return, so it will be empty
-
- There are several default arguments that affect the scan or the
- return values. XXX should document them.
-
- The file position returned is the position just after the last
- valid transaction record. The oid returned is the maximum object
- id in the data. The transaction id is the tid of the last
- transaction.
- """
- read = file.read
- seek = file.seek
- seek(0, 2)
- file_size=file.tell()
-
- if file_size:
- if file_size < start: raise FileStorageFormatError, file.name
- seek(0)
- if read(4) != packed_version: raise FileStorageFormatError, name
- else:
- if not read_only: file.write(packed_version)
- return 4L, maxoid, ltid
-
- index_get=index.get
-
- pos=start
- seek(start)
- tid='\0'*7+'\1'
-
- while 1:
- # Read the transaction record
- h=read(TRANS_HDR_LEN)
- if not h: break
- if len(h) != TRANS_HDR_LEN:
- if not read_only:
- warn('%s truncated at %s', name, pos)
- seek(pos)
- file.truncate()
- break
-
- tid, tl, status, ul, dl, el = unpack(TRANS_HDR,h)
- if el < 0: el=t32-el
-
- if tid <= ltid:
- warn("%s time-stamp reduction at %s", name, pos)
- ltid=tid
-
- if pos+(tl+8) > file_size or status=='c':
- # Hm, the data were truncated or the checkpoint flag wasn't
- # cleared. They may also be corrupted,
- # in which case, we don't want to totally lose the data.
- if not read_only:
- warn("%s truncated, possibly due to damaged records at %s",
- name, pos)
- _truncate(file, name, pos)
- break
-
- if status not in ' up':
- warn('%s has invalid status, %s, at %s', name, status, pos)
-
- if tl < (TRANS_HDR_LEN+ul+dl+el):
- # We're in trouble. Find out if this is bad data in the
- # middle of the file, or just a turd that Win 9x dropped
- # at the end when the system crashed.
- # Skip to the end and read what should be the transaction length
- # of the last transaction.
- seek(-8, 2)
- rtl=u64(read(8))
- # Now check to see if the redundant transaction length is
- # reasonable:
- if file_size - rtl < pos or rtl < TRANS_HDR_LEN:
- nearPanic('%s has invalid transaction header at %s', name, pos)
- if not read_only:
- warn("It appears that there is invalid data at the end of "
- "the file, possibly due to a system crash. %s "
- "truncated to recover from bad data at end.",
- name)
- _truncate(file, name, pos)
- break
- else:
- if recover: return pos, None, None
- panic('%s has invalid transaction header at %s', name, pos)
-
- if tid >= stop: break
-
- tpos=pos
- tend=tpos+tl
-
- if status=='u':
- # Undone transaction, skip it
- seek(tend)
- h=read(8)
- if h != stl:
- if recover: return tpos, None, None
- panic('%s has inconsistent transaction length at %s',
- name, pos)
- pos=tend+8
- continue
-
- pos=tpos+(TRANS_HDR_LEN+ul+dl+el)
- while pos < tend:
- # Read the data records for this transaction
-
- seek(pos)
- h=read(DATA_HDR_LEN)
- oid,serial,prev,tloc,vlen,plen = unpack(DATA_HDR, h)
- dlen=DATA_HDR_LEN+(plen or 8)
- tindex[oid]=pos
-
- if vlen:
- dlen=dlen+(16+vlen)
- read(16)
- version=read(vlen)
- vindex[version]=pos
-
- if pos+dlen > tend or tloc != tpos:
- if recover: return tpos, None, None
- panic("%s data record exceeds transaction record at %s",
- name, pos)
-
- if index_get(oid, 0) != prev:
- if prev:
- if recover: return tpos, None, None
- error("%s incorrect previous pointer at %s", name, pos)
- else:
- warn("%s incorrect previous pointer at %s", name, pos)
-
- pos=pos+dlen
-
- if pos != tend:
- if recover: return tpos, None, None
- panic("%s data records don't add up at %s",name,tpos)
-
- # Read the (intentionally redundant) transaction length
- seek(pos)
- l = u64(read(8))
- if l != tl:
- if recover: return tpos, None, None
- panic("%s redundant transaction length check failed at %s",
- name, pos)
- pos += 8
-
- if tindex: # avoid the pathological empty transaction case
- _maxoid = max(tindex.keys()) # in 2.2, just max(tindex)
- maxoid = max(_maxoid, maxoid)
- index.update(tindex)
- tindex.clear()
-
- return pos, maxoid, ltid
def _truncate(file, name, pos):
=== Zope3/src/zodb/storage/interfaces.py 1.6 => 1.7 ===
--- Zope3/src/zodb/storage/interfaces.py:1.6 Tue Feb 11 10:59:27 2003
+++ Zope3/src/zodb/storage/interfaces.py Tue Feb 25 13:55:03 2003
@@ -169,7 +169,10 @@
class IUndoStorage(Interface):
def loadSerial(oid, serial):
- pass
+ """Return data record for revision `serial` of `oid.`
+
+ Raises POSKeyError if the revisions is not available.
+ """
def undo(txnid, txn):
pass
=== Zope3/src/zodb/storage/mapping.py 1.4 => 1.5 ===
--- Zope3/src/zodb/storage/mapping.py:1.4 Tue Feb 11 10:59:27 2003
+++ Zope3/src/zodb/storage/mapping.py Tue Feb 25 13:55:03 2003
@@ -1,6 +1,6 @@
##############################################################################
#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# Copyright (c) 2001 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
@@ -18,113 +18,42 @@
It is meant to illustrate the simplest possible storage.
-The Mapping storage uses a single data structure to map
-object ids to data.
-
-The Demo storage serves two purposes:
-
- - Provide an example implementation of a full storage without
- distracting storage details,
-
- - Provide a volatile storage that is useful for giving demonstrations.
-
-The demo strorage can have a "base" storage that is used in a
-read-only fashion. The base storage must not not to contain version
+The Mapping storage uses a single data structure to map object ids to
data.
-
-There are three main data structures:
-
- _data -- Transaction logging information necessary for undo
-
- This is a mapping from transaction id to transaction, where
- a transaction is simply a 4-tuple:
-
- packed, user, description, extension_data, records
-
- where extension_data is a dictionary or None and records are the
- actual records in chronological order. Packed is a flag
- indicating whethe the transaction has been packed or not
-
- _index -- A mapping from oid to record
-
- _vindex -- A mapping from version name to version data
-
- where version data is a mapping from oid to record
-
-A record is a tuple:
-
- oid, serial, pre, vdata, p,
-
-where:
-
- oid -- object id
-
- serial -- object serial number
-
- pre -- The previous record for this object (or None)
-
- vdata -- version data
-
- None if not a version, ortherwise:
- version, non-version-record
-
- p -- the pickle data or None
-
-The pickle data will be None for a record for an object created in
-an aborted version.
-
-It is instructive to watch what happens to the internal data structures
-as changes are made. Foe example, in Zope, you can create an external
-method::
-
- import Zope
-
- def info(RESPONSE):
- RESPONSE['Content-type']= 'text/plain'
-
- return Zope.DB._storage._splat()
-
-and call it to minotor the storage.
-
$Id$
"""
import zodb.db
from zodb import interfaces, utils
-from zodb.storage import base
+from zodb.storage.base import BaseStorage
from zodb.storage.interfaces import *
from zodb.serialize import findrefs
from zodb.timestamp import TimeStamp
from zodb.utils import z64
-def DB(name="Mapping Storage",
- pool_size=7, cache_size=400):
- ms = MappingStorage(name)
- db = zodb.db.DB(ms, pool_size, cache_size)
- return db
-
-class MappingStorage(base.BaseStorage):
+class MappingStorage(BaseStorage):
__implements__ = IStorage
- def __init__(self, name='Mapping Storage'):
-
- base.BaseStorage.__init__(self, name)
+ def __init__(self, name="Mapping Storage"):
+ BaseStorage.__init__(self, name)
+ self._index = {}
+ self._tindex = []
+ self._ltid = None
- self._index={}
- self._tindex=[]
+ def close(self):
+ pass
- # Note:
- # If you subclass this and use a persistent mapping facility
- # (e.g. a dbm file), you will need to get the maximum key and
- # save it as self._oid. See dbmStorage.
+ def cleanup(self):
+ pass
def load(self, oid, version):
self._lock_acquire()
try:
- p=self._index[oid]
+ p = self._index[oid]
return p[8:], p[:8] # pickle, serial
- finally: self._lock_release()
+ finally:
+ self._lock_release()
def store(self, oid, serial, data, version, transaction):
if transaction is not self._transaction:
@@ -136,61 +65,61 @@
self._lock_acquire()
try:
if self._index.has_key(oid):
- old=self._index[oid]
- oserial=old[:8]
+ old = self._index[oid]
+ oserial = old[:8]
if serial != oserial:
raise interfaces.ConflictError(serials=(oserial, serial))
- serial=self._serial
- self._tindex.append((oid,serial+data))
+ self._tindex.append((oid, self._serial + data))
finally: self._lock_release()
- return serial
+ return self._serial
def _clear_temp(self):
- self._tindex=[]
+ self._tindex = []
def _finish(self, tid, user, desc, ext):
+ for oid, p in self._tindex:
+ self._index[oid] = p
+ self._ltid = self._serial
- index=self._index
- for oid, p in self._tindex: index[oid]=p
+ def lastTransaction(self):
+ return self._ltid
def pack(self, t):
self._lock_acquire()
try:
- # Build an index of *only* those objects reachable
- # from the root.
- index=self._index
- rootl = [z64]
- pop=rootl.pop
- pindex={}
- referenced=pindex.has_key
+ # Build an index of *only* those objects reachable from the root.
+ rootl = ["\0\0\0\0\0\0\0\0"]
+ pindex = {}
while rootl:
- oid=pop()
- if referenced(oid): continue
-
+ oid = rootl.pop()
+ if pindex.has_key(oid):
+ continue
# Scan non-version pickle for references
- r=index[oid]
- pindex[oid]=r
- p=r[8:]
+ r = self._index[oid]
+ pindex[oid] = r
+ p = r[8:]
rootl.extend(findrefs(p))
# Now delete any unreferenced entries:
- for oid in index.keys():
- if not referenced(oid): del index[oid]
+ for oid in self._index.keys():
+ if not pindex.has_key(oid):
+ del self._index[oid]
- finally: self._lock_release()
+ finally:
+ self._lock_release()
def _splat(self):
"""Spit out a string showing state."""
o=[]
- o.append('Index:')
+ o.append("Index:")
index=self._index
keys=index.keys()
keys.sort()
for oid in keys:
r=index[oid]
- o.append(' %s: %s, %s' %
+ o.append(" %s: %s, %s" %
(utils.u64(oid),TimeStamp(r[:8]),`r[8:]`))
return "\n".join(o)