[Zope-Checkins] CVS: Zope/lib/python/Products/Transience - TransientObject.py:1.1 Transience.py:1.21 TransienceInterfaces.py:1.10 __init__.py:1.5
Chris McDonough
chrism@zope.com
Wed, 21 Nov 2001 17:47:07 -0500
Update of /cvs-repository/Zope/lib/python/Products/Transience
In directory cvs.zope.org:/tmp/cvs-serv11583
Modified Files:
Transience.py TransienceInterfaces.py __init__.py
Added Files:
TransientObject.py
Log Message:
Moved TransientObjects into their own module.
Removed wrap_with argument from new and new_or_existing methods
of Transient Data Containers.
Removed delete method of Transient Data Containers.
Added out-of-memory protection to Transient Data Containers. A
new __init__ value ('limit') is used to specify the max number
of objects that can be contained within a transient data container.
A new envvar ZSESSION_OBJECT_LIMIT can be used to control the
limit of the default session_data TDC. Also updated help and
API docs with this change.
Added a new exception, MaxTransientObjectsExceeded, which is raised
when the OOM protection kicks in.
Various implementation changes including the use of a BTrees Length
object to store Transient Data Container length info as well
as improvements to how buckets are expired.
Addition of tests for OOM protection fatures.
=== Added File Zope/lib/python/Products/Transience/TransientObject.py ===
##############################################################################
#
# Zope Public License (ZPL) Version 1.0
# -------------------------------------
#
# Copyright (c) Digital Creations. All rights reserved.
#
# This license has been certified as Open Source(tm).
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions in source code must retain the above copyright
# notice, this list of conditions, and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions, and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# 3. Digital Creations requests that attribution be given to Zope
# in any manner possible. Zope includes a "Powered by Zope"
# button that is installed by default. While it is not a license
# violation to remove this button, it is requested that the
# attribution remain. A significant investment has been put
# into Zope, and this effort will continue if the Zope community
# continues to grow. This is one way to assure that growth.
#
# 4. All advertising materials and documentation mentioning
# features derived from or use of this software must display
# the following acknowledgement:
#
# "This product includes software developed by Digital Creations
# for use in the Z Object Publishing Environment
# (http://www.zope.org/)."
#
# In the event that the product being advertised includes an
# intact Zope distribution (with copyright and license included)
# then this clause is waived.
#
# 5. Names associated with Zope or Digital Creations must not be used to
# endorse or promote products derived from this software without
# prior written permission from Digital Creations.
#
# 6. Modified redistributions of any form whatsoever must retain
# the following acknowledgment:
#
# "This product includes software developed by Digital Creations
# for use in the Z Object Publishing Environment
# (http://www.zope.org/)."
#
# Intact (re-)distributions of any official Zope release do not
# require an external acknowledgement.
#
# 7. Modifications are encouraged but must be packaged separately as
# patches to official Zope releases. Distributions that do not
# clearly separate the patches from the original work must be clearly
# labeled as unofficial distributions. Modifications which do not
# carry the name Zope may be packaged in any form, as long as they
# conform to all of the clauses above.
#
#
# Disclaimer
#
# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
#
# This software consists of contributions made by Digital Creations and
# many individuals on behalf of Digital Creations. Specific
# attributions are listed in the accompanying credits file.
#
##############################################################################
"""
Simple ZODB-based transient object implementation.
$Id: TransientObject.py,v 1.1 2001/11/21 22:46:36 chrism Exp $
"""
__version__='$Revision: 1.1 $'[11:-2]
from Persistence import Persistent
from Acquisition import Implicit, aq_base
import time, random, sys
from TransienceInterfaces import ItemWithId, Transient, DictionaryLike,\
TTWDictionary, ImmutablyValuedMappingOfPickleableObjects
from AccessControl import ClassSecurityInfo
import Globals
from zLOG import LOG, BLATHER
_notfound = []
WRITEGRANULARITY=30 # Timing granularity for write clustering, in seconds
class TransientObject(Persistent, Implicit):
""" Dictionary-like object that supports additional methods
concerning expiration and containment in a transient object container
"""
__implements__ = (ItemWithId, # randomly generate an id
Transient,
DictionaryLike,
TTWDictionary,
ImmutablyValuedMappingOfPickleableObjects
)
security = ClassSecurityInfo()
security.setDefaultAccess('allow')
security.declareObjectPublic()
def __init__(self, containerkey):
self.token = containerkey
self.id = self._generateUniqueId()
self._container = {}
self._created = self._last_accessed = time.time()
# -----------------------------------------------------------------
# ItemWithId
#
def getId(self):
return self.id
# -----------------------------------------------------------------
# Transient
#
def invalidate(self):
self._invalid = None
def isValid(self):
return not hasattr(self, '_invalid')
def getLastAccessed(self):
return self._last_accessed
def setLastAccessed(self, WG=WRITEGRANULARITY):
# check to see if the last_accessed time is too recent, and avoid
# setting if so, to cut down on heavy writes
t = time.time()
if (self._last_accessed + WG) < t:
self._last_accessed = t
def getCreated(self):
return self._created
def getContainerKey(self):
return self.token
# -----------------------------------------------------------------
# DictionaryLike
#
def keys(self):
return self._container.keys()
def values(self):
return self._container.values()
def items(self):
return self._container.items()
def get(self, k, default=_notfound):
v = self._container.get(k, default)
if v is _notfound: return None
return v
def has_key(self, k):
if self._container.get(k, _notfound) is not _notfound: return 1
return 0
def clear(self):
self._container.clear()
self._p_changed = 1
def update(self, d):
for k in d.keys():
self[k] = d[k]
# -----------------------------------------------------------------
# ImmutablyValuedMappingOfPickleableObjects (what a mouthful!)
#
def __setitem__(self, k, v):
# if the key or value is a persistent instance,
# set up its _p_jar immediately
if hasattr(v, '_p_jar') and v._p_jar is None:
v._p_jar = self._p_jar
v._p_changed = 1
if hasattr(k, '_p_jar') and k._p_jar is None:
k._p_jar = self._p_jar
k._p_changed = 1
self._container[k] = v
self._p_changed = 1
def __getitem__(self, k):
return self._container[k]
def __delitem__(self, k):
del self._container[k]
# -----------------------------------------------------------------
# TTWDictionary
#
set = __setitem__
def delete(self, k):
del self._container[k]
self._p_changed = 1
__guarded_setitem__ = __setitem__
# -----------------------------------------------------------------
# Other non interface code
#
def _p_independent(self):
# My state doesn't depend on or materially effect the state of
# other objects (eliminates read conflicts).
return 1
def _p_resolveConflict(self, saved, state1, state2):
attrs = ['token', 'id', '_created', '_invalid']
# note that last_accessed and _container are the only attrs
# missing from this list. The only time we can clearly resolve
# the conflict is if everything but the last_accessed time and
# the contents are the same, so we make sure nothing else has
# changed. We're being slightly sneaky here by accepting
# possibly conflicting data in _container, but it's acceptable
# in this context.
LOG('Transience', BLATHER, 'Resolving conflict in TransientObject')
for attr in attrs:
old = saved.get(attr)
st1 = state1.get(attr)
st2 = state2.get(attr)
if not (old == st1 == st2):
return None
# return the object with the most recent last_accessed value.
if state1['_last_accessed'] > state2['_last_accessed']:
return state1
else:
return state2
getName = getId # this is for SQLSession compatibility
def _generateUniqueId(self):
t = str(int(time.time()))
d = "%010d" % random.randint(0, sys.maxint-1)
return "%s%s" % (t, d)
def __repr__(self):
return "id: %s, token: %s, contents: %s" % (
self.id, self.token, `self.items()`
)
Globals.InitializeClass(TransientObject)
=== Zope/lib/python/Products/Transience/Transience.py 1.20 => 1.21 === (668/768 lines abridged)
##############################################################################
"""
-Core session tracking SessionData class.
+Transient Object Container class.
$Id$
"""
@@ -92,40 +92,33 @@
import Globals
from Globals import HTMLFile, MessageDialog
-from TransienceInterfaces import Transient, DictionaryLike, ItemWithId,\
- TTWDictionary, ImmutablyValuedMappingOfPickleableObjects,\
+from TransienceInterfaces import ItemWithId,\
StringKeyedHomogeneousItemContainer, TransientItemContainer
+from TransientObject import TransientObject
from OFS.SimpleItem import SimpleItem
-from Persistence import Persistent, PersistentMapping
-from Acquisition import Implicit, aq_base
+from Persistence import Persistent
from AccessControl import ClassSecurityInfo, getSecurityManager
from AccessControl.SecurityManagement import newSecurityManager
import AccessControl.SpecialUsers
from AccessControl.User import nobody
from BTrees import OOBTree
+from BTrees.Length import Length
from zLOG import LOG, WARNING, BLATHER
-import os
-import os.path
-import math
-import time
-import sys
-import random
-from types import InstanceType
+import os, os.path, math, time, sys, random
DEBUG = os.environ.get('Z_TOC_DEBUG', '')
-def TLOG(*args):
+def DLOG(*args):
tmp = []
for arg in args:
tmp.append(str(arg))
LOG('Transience DEBUG', BLATHER, ' '.join(tmp))
+class MaxTransientObjectsExceeded(Exception): pass
+
_notfound = []
_marker = []
[-=- -=- -=- 668 lines omitted -=- -=- -=-]
-
- def __getitem__(self, k):
- return self._container[k]
-
- def __delitem__(self, k):
- del self._container[k]
-
- # -----------------------------------------------------------------
- # TTWDictionary
- #
-
- set = __setitem__
-
- def delete(self, k):
- del self._container[k]
- self._p_changed = 1
-
- __guarded_setitem__ = __setitem__
-
-
- # -----------------------------------------------------------------
- # Other non interface code
- #
-
- def _p_independent(self):
- # My state doesn't depend on or materially effect the state of
- # other objects (eliminates read conflicts).
- return 1
-
- getName = getId # this is for SQLSession compatibility
-
- def getContainerKey(self):
- return self.token
-
- def _generateUniqueId(self):
- t = str(int(time()))
- d = "%010d" % random.randint(0, sys.maxint-1)
- return "%s%s" % (t, d)
-
- def __repr__(self):
- return "id: %s, token: %s, contents: %s" % (
- self.id, self.token, `self.items()`
- )
+ # this should really have a _p_resolveConflict, but
+ # I've not had time to come up with a reasonable one that
+ # works in every circumstance.
Globals.InitializeClass(TransientObjectContainer)
-Globals.InitializeClass(TransientObject)
+
=== Zope/lib/python/Products/Transience/TransienceInterfaces.py 1.9 => 1.10 ===
Return value associated with key k via __getitem__. If value
associated with k does not exist, return default.
+
+ Returned item is acquisition-wrapped in self unless a default
+ is passed in and returned.
"""
def has_key(self, k):
@@ -283,13 +286,8 @@
return false.
"""
- def delete(self, k):
- """
- Delete value associated with key k, raise a KeyError if nonexistent.
- """
-
class StringKeyedHomogeneousItemContainer(HomogeneousItemContainer):
- def new(self, k, wrap_with=None):
+ def new(self, k):
"""
Creates a new subobject of the type supported by this container
with key "k" and returns it.
@@ -297,14 +295,15 @@
If an object already exists in the container with key "k", a
KeyError is raised.
- If wrap_with is non-None, the subobject is returned in the
- acquisition context of wrap_in, else it is returned in
- the acquisition context of this transient object container.
-
"k" must be a string, else a TypeError is raised.
+
+ If the container is 'full', a MaxTransientObjectsExceeded exception
+ will be raised.
+
+ Returned object is acquisition-wrapped in self.
"""
- def new_or_existing(self, k, wrap_with=None):
+ def new_or_existing(self, k):
"""
If an object already exists in the container with key "k", it
is returned.
@@ -312,11 +311,12 @@
Otherwise, create a new subobject of the type supported by this
container with key "k" and return it.
- If wrap_with is non-None, the subobject is returned in the
- acquisition context of wrap_in, else it is returned in
- the acquisition context of this transient object container.
-
"k" must be a string, else a TypeError is raised.
+
+ If a new object needs to be created and the container is 'full',
+ a MaxTransientObjectsExceeded exception will be raised.
+
+ Returned object is acquisition-wrapped in self.
"""
class TransientItemContainer(Interface.Base):
=== Zope/lib/python/Products/Transience/__init__.py 1.4 => 1.5 ===
"""
+import ZODB # this is to help out testrunner, don't remove.
import Transience
+# import of MaxTransientObjectsExceeded for easy import from scripts,
+# this is protected by a module security info declaration in the
+# Sessions package.
+from Transience import MaxTransientObjectsExceeded
def initialize(context):
context.registerClass(
@@ -100,3 +105,4 @@
)
context.registerHelp()
context.registerHelpTitle('Zope Help')
+