[Zope-Checkins] CVS: Zope/lib/python/ZODB - Connection.py:1.64 DB.py:1.40 cPersistence.c:1.51 cPersistence.h:1.22 cPickleCache.c:1.42
Toby Dickenson
tdickenson@geminidataloggers.com
Wed, 27 Mar 2002 05:14:05 -0500
Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv10497/lib/python/ZODB
Modified Files:
Connection.py DB.py cPersistence.c cPersistence.h
cPickleCache.c
Log Message:
merged toby-stiff-cache-branch and toby-unicode-branch
=== Zope/lib/python/ZODB/Connection.py 1.63 => 1.64 ===
__version__='$Revision$'[11:-2]
-from cPickleCache import PickleCache
+from cPickleCache import PickleCache, MUCH_RING_CHECKING
from POSException import ConflictError, ReadConflictError
from ExtensionClass import Base
import ExportImport, TmpStore
-from zLOG import LOG, ERROR, BLATHER
+from zLOG import LOG, ERROR, BLATHER, WARNING
from coptimizations import new_persistent_id
from ConflictResolution import ResolvedSerial
@@ -32,6 +32,11 @@
global_code_timestamp = 0
+if MUCH_RING_CHECKING:
+ # To get rid of this warning, change the define inside cPickleCache.c and recompile.
+ LOG('ZODB',WARNING, 'Using cPickleCache with low performance (but extra debugging checks)')
+del MUCH_RING_CHECKING
+
def updateCodeTimestamp():
'''
Called after changes are made to persistence-based classes.
@@ -65,12 +70,35 @@
"""Create a new Connection"""
self._version=version
self._cache=cache=PickleCache(self, cache_size, cache_deactivate_after)
+ if version:
+ # Caches for versions end up empty if the version
+ # is not used for a while. Non-version caches
+ # keep their content indefinitely.
+ self._cache.cache_drain_resistance = 100
self._incrgc=self.cacheGC=cache.incrgc
self._invalidated=d={}
self._invalid=d.has_key
self._committed=[]
self._code_timestamp = global_code_timestamp
+ def _cache_items(self):
+ # find all items on the lru list
+ items = self._cache.lru_items()
+ # fine everything. some on the lru list, some not
+ everything = self._cache.cache_data
+ # remove those items that are on the lru list
+ for k,v in items:
+ del everything[k]
+ # return a list of [ghosts....not recently used.....recently used]
+ return everything.items() + items
+
+ def __repr__(self):
+ if self._version:
+ ver = ' (in version %s)' % `self._version`
+ else:
+ ver = ''
+ return '<Connection at %08x%s>' % (id(self),ver)
+
def _breakcr(self):
try: del self._cache
except: pass
@@ -414,9 +442,9 @@
for oid in creating:
o=cache_get(oid, None)
if o is not None:
+ del cache[oid]
del o._p_jar
del o._p_oid
- del cache[oid]
#XXX
@@ -441,9 +469,14 @@
def root(self): return self['\0\0\0\0\0\0\0\0']
def setstate(self, object):
+ oid=object._p_oid
+
+ if self._storage is None:
+ msg = "Shouldn't load state for %s when the connection is closed" % `oid`
+ LOG('ZODB',ERROR, msg)
+ raise RuntimeError(msg)
+
try:
- oid=object._p_oid
-
p, serial = self._storage.load(oid, self._version)
# XXX this is quite conservative!
=== Zope/lib/python/ZODB/DB.py 1.39 => 1.40 ===
conn_no[0] = conn_no[0] + 1
cn = conn_no[0]
- for oid, ob in con._cache.items():
+ for oid, ob in con._cache_items():
id=''
if hasattr(ob,'__dict__'):
d=ob.__dict__
@@ -224,10 +224,20 @@
def cacheSize(self):
m=[0]
def f(con, m=m):
- m[0]=m[0]+len(con._cache)
+ m[0]=m[0]+con._cache.cache_non_ghost_count
self._connectionMap(f)
return m[0]
+
+ def cacheDetailSize(self):
+ m=[]
+ def f(con, m=m):
+ m.append({'connection':repr(con),
+ 'ngsize':con._cache.cache_non_ghost_count,
+ 'size':len(con._cache)})
+ self._connectionMap(f)
+ m.sort()
+ return m
def close(self): self._storage.close()
=== Zope/lib/python/ZODB/cPersistence.c 1.50 => 1.51 ===
#include "cPersistence.h"
+/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
+struct ccobject_head_struct {
+ PyObject_HEAD
+ CPersistentRing ring_home;
+ int non_ghost_count;
+};
+
+#define HOME(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
+#define NON_GHOST_COUNT(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
+
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
@@ -112,21 +122,82 @@
{ \
PyObject *r; \
\
+ int *count = NON_GHOST_COUNT(self); \
+ if(count) \
+ { \
+ (*count)++; \
+ self->ring.next = HOME(self); \
+ self->ring.prev = HOME(self)->prev; \
+ HOME(self)->prev->next = &self->ring; \
+ HOME(self)->prev = &self->ring; \
+ Py_INCREF(self); \
+ } \
self->state=cPersistent_CHANGED_STATE; \
UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \
{ \
- self->state=cPersistent_GHOST_STATE; \
+ ghostify(self); \
return ER; \
} \
self->state=cPersistent_UPTODATE_STATE; \
Py_DECREF(r); \
}
+#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
+ if(HOME(self) && self->state>=0) { \
+ self->ring.prev->next = self->ring.next; \
+ self->ring.next->prev = self->ring.prev; \
+ self->ring.next = HOME(self); \
+ self->ring.prev = HOME(self)->prev; \
+ HOME(self)->prev->next = &self->ring; \
+ HOME(self)->prev = &self->ring; }
+
+
/****************************************************************************/
staticforward PyExtensionClass Pertype;
+static void
+accessed(cPersistentObject *self)
+{
+ KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
+}
+
+static void
+ghostify(cPersistentObject *self)
+{
+ int *count;
+ count = NON_GHOST_COUNT(self);
+ if(count && (self->state>=0))
+ {
+ (*count)--;
+ self->ring.next->prev = self->ring.prev;
+ self->ring.prev->next = self->ring.next;
+ self->ring.prev = NULL;
+ self->ring.next = NULL;
+ self->state = cPersistent_GHOST_STATE;
+ Py_DECREF(self);
+ }
+ else
+ {
+ self->state = cPersistent_GHOST_STATE;
+ }
+}
+
+static void
+deallocated(cPersistentObject *self)
+{
+ if(self->state>=0) ghostify(self);
+ if(self->cache)
+ {
+ PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
+ if(!v) PyErr_Clear(); /* and explode later */
+ Py_XDECREF(v);
+ }
+ Py_XDECREF(self->jar);
+ Py_XDECREF(self->oid);
+}
+
static int
changed(cPersistentObject *self)
{
@@ -185,7 +256,7 @@
static PyObject *
Per__p_deactivate(cPersistentObject *self, PyObject *args)
{
- PyObject *dict;
+ PyObject *dict,*dict2=NULL;
#ifdef DEBUG_LOG
if (idebug_log < 0) call_debug("reinit",self);
@@ -197,13 +268,22 @@
if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
HasInstDict(self) && (dict=INSTANCE_DICT(self)))
{
+ dict2 = PyDict_Copy(dict);
PyDict_Clear(dict);
/* Note that we need to set to ghost state unless we are
called directly. Methods that override this need to
do the same! */
- self->state=cPersistent_GHOST_STATE;
+ ghostify(self);
}
+ /* need to delay releasing the last reference on instance attributes
+ until after we have finished accounting for losing our state */
+ if(dict2)
+ {
+ PyDict_Clear(dict2);
+ Py_DECREF(dict2);
+ }
+
Py_INCREF(Py_None);
return Py_None;
}
@@ -333,8 +413,8 @@
#ifdef DEBUG_LOG
if(idebug_log < 0) call_debug("del",self);
#endif
- Py_XDECREF(self->jar);
- Py_XDECREF(self->oid);
+ deallocated(self);
+ Py_XDECREF(self->cache);
Py_DECREF(self->ob_type);
PyObject_DEL(self);
}
@@ -387,7 +467,7 @@
{
UPDATE_STATE_IF_NECESSARY(self, NULL);
- self->atime=((long)(time(NULL)/3))%65536;
+ KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
self->serial[5]=='\0' && self->serial[4]=='\0' &&
@@ -419,7 +499,7 @@
{
UPDATE_STATE_IF_NECESSARY(self, NULL);
- self->atime=((long)(time(NULL)/3))%65536;
+ KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
return getattrf((PyObject *)self, oname);
@@ -466,6 +546,21 @@
{
if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
{
+ if(HOME(self))
+ {
+ int result;
+ if(!v)
+ {
+ PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
+ return -1;
+ }
+ if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
+ if(result)
+ {
+ PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
+ return -1;
+ }
+ }
Py_XINCREF(v);
ASSIGN(self->oid, v);
return 0;
@@ -509,7 +604,6 @@
v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
if (v) { Py_DECREF(v); }
- self->state=cPersistent_GHOST_STATE;
return 0;
}
if (PyObject_IsTrue(v)) return changed(self);
@@ -521,8 +615,7 @@
{
UPDATE_STATE_IF_NECESSARY(self, -1);
- /* Record access times */
- self->atime=((long)(time(NULL)/3))%65536;
+ KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
&& (self->state != cPersistent_CHANGED_STATE && self->jar)
@@ -680,9 +773,11 @@
(getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed,
+ accessed,
+ ghostify,
+ deallocated,
(intfunctionwithpythonarg)Per_setstate,
(pergetattr)Per_getattr,
- (persetattr)_setattro,
};
void
=== Zope/lib/python/ZODB/cPersistence.h 1.21 => 1.22 ===
#include <time.h>
-#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime; signed char state; unsigned char reserved;
+
+#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
#define cPersistent_GHOST_STATE -1
#define cPersistent_UPTODATE_STATE 0
#define cPersistent_CHANGED_STATE 1
#define cPersistent_STICKY_STATE 2
+struct ccobject_head_struct;
+
+typedef struct CPersistentRing_struct
+{
+ struct CPersistentRing_struct *prev;
+ struct CPersistentRing_struct *next;
+} CPersistentRing;
+
typedef struct {
cPersistent_HEAD
} cPersistentObject;
@@ -36,6 +45,9 @@
getattrofunc getattro;
setattrofunc setattro;
int (*changed)(cPersistentObject*);
+ void (*accessed)(cPersistentObject*);
+ void (*ghostify)(cPersistentObject*);
+ void (*deallocated)(cPersistentObject*);
int (*setstate)(PyObject*);
pergetattr pergetattro;
persetattr persetattro;
@@ -59,11 +71,13 @@
#define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
-#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
+#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
+
+#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
-#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
+#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
-#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid);
+#define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
#define PER_USE(O) \
(((O)->state != cPersistent_GHOST_STATE \
@@ -71,7 +85,7 @@
? (((O)->state==cPersistent_UPTODATE_STATE) \
? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
-#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536)
+#define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
#endif
=== Zope/lib/python/ZODB/cPickleCache.c 1.41 => 1.42 === (1287/1387 lines abridged)
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
-
+
This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
-
+
****************************************************************************/
-static char cPickleCache_doc_string[] =
+
+static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n"
"\n"
"$Id$\n";
-/* Compute the current time in the units and range used for peristent
- objects. */
-#define PER_TIME() ((long)(time(NULL) / 3)) % 65536
+#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
+#define UNLESS(E) if(!(E))
+#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
+#define OBJECT(O) ((PyObject*)O)
#define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h"
#include <time.h>
+#include <stddef.h>
#undef Py_FindMethod
-static PyObject *py_reload, *py__p_jar, *py__p_changed;
-typedef struct {
- PyObject_HEAD
- PyObject *data;
- PyObject *jar;
- PyObject *setklassstate;
- int position;
- int cache_size;
- int cache_age;
- /* Cache statistics */
- int sum_deal;
- int sum_deac;
- double sum_age;
- int n, na;
[-=- -=- -=- 1287 lines omitted -=- -=- -=-]
+ /* Space for future expansion */
+ 0L,0L,0L,0L,
+ ""
};
static PyObject *
@@ -644,9 +924,9 @@
int cache_size=100, cache_age=1000;
PyObject *jar;
- if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
+ UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
return NULL;
- return (PyObject *)newccobject(jar, cache_size, cache_age);
+ return (PyObject*)newccobject(jar, cache_size,cache_age);
}
static struct PyMethodDef cCM_methods[] = {
@@ -657,12 +937,11 @@
void
initcPickleCache(void)
{
- PyObject *m;
+ PyObject *m, *d;
- Cctype.ob_type = &PyType_Type;
+ Cctype.ob_type=&PyType_Type;
- if (!ExtensionClassImported)
- return;
+ UNLESS(ExtensionClassImported) return;
m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
(PyObject*)NULL, PYTHON_API_VERSION);
@@ -670,4 +949,15 @@
py_reload = PyString_InternFromString("reload");
py__p_jar = PyString_InternFromString("_p_jar");
py__p_changed = PyString_InternFromString("_p_changed");
+ py__p_oid = PyString_InternFromString("_p_oid");
+
+ d = PyModule_GetDict(m);
+
+ PyDict_SetItemString(d,"cache_variant",PyString_FromString("stiff/c"));
+
+#ifdef MUCH_RING_CHECKING
+ PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(1));
+#else
+ PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(0));
+#endif
}