[Zope-Checkins] CVS: Zope/lib/python/ZODB - Connection.py:1.64 DB.py:1.40 cPersistence.c:1.51 cPersistence.h:1.22 cPickleCache.c:1.42

Toby Dickenson tdickenson@geminidataloggers.com
Wed, 27 Mar 2002 05:14:05 -0500


Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv10497/lib/python/ZODB

Modified Files:
	Connection.py DB.py cPersistence.c cPersistence.h 
	cPickleCache.c 
Log Message:
merged toby-stiff-cache-branch and toby-unicode-branch

=== Zope/lib/python/ZODB/Connection.py 1.63 => 1.64 ===
 __version__='$Revision$'[11:-2]
 
-from cPickleCache import PickleCache
+from cPickleCache import PickleCache, MUCH_RING_CHECKING
 from POSException import ConflictError, ReadConflictError
 from ExtensionClass import Base
 import ExportImport, TmpStore
-from zLOG import LOG, ERROR, BLATHER
+from zLOG import LOG, ERROR, BLATHER, WARNING
 from coptimizations import new_persistent_id
 from ConflictResolution import ResolvedSerial
 
@@ -32,6 +32,11 @@
 
 global_code_timestamp = 0
 
+if MUCH_RING_CHECKING:
+  # To get rid of this warning, change the define inside cPickleCache.c and recompile.
+  LOG('ZODB',WARNING, 'Using cPickleCache with low performance (but extra debugging checks)')
+del MUCH_RING_CHECKING
+
 def updateCodeTimestamp():
     '''
     Called after changes are made to persistence-based classes.
@@ -65,12 +70,35 @@
         """Create a new Connection"""
         self._version=version
         self._cache=cache=PickleCache(self, cache_size, cache_deactivate_after)
+        if version:
+            # Caches for versions end up empty if the version
+            # is not used for a while. Non-version caches
+            # keep their content indefinitely.
+            self._cache.cache_drain_resistance = 100
         self._incrgc=self.cacheGC=cache.incrgc
         self._invalidated=d={}
         self._invalid=d.has_key
         self._committed=[]
         self._code_timestamp = global_code_timestamp
 
+    def _cache_items(self):
+        # find all items on the lru list
+        items = self._cache.lru_items()
+        # fine everything. some on the lru list, some not
+        everything = self._cache.cache_data
+        # remove those items that are on the lru list
+        for k,v in items:
+            del everything[k]
+        # return a list of [ghosts....not recently used.....recently used]
+        return everything.items() + items
+
+    def __repr__(self):
+        if self._version:
+            ver = ' (in version %s)' % `self._version`
+        else:
+            ver = ''
+        return '<Connection at %08x%s>' % (id(self),ver)
+
     def _breakcr(self):
         try: del self._cache
         except: pass
@@ -414,9 +442,9 @@
         for oid in creating:
             o=cache_get(oid, None)
             if o is not None:
+                del cache[oid]
                 del o._p_jar
                 del o._p_oid
-                del cache[oid]
 
     #XXX
 
@@ -441,9 +469,14 @@
     def root(self): return self['\0\0\0\0\0\0\0\0']
 
     def setstate(self, object):
+        oid=object._p_oid
+
+        if self._storage is None:
+            msg = "Shouldn't load state for %s when the connection is closed" % `oid`
+            LOG('ZODB',ERROR, msg)
+            raise RuntimeError(msg)
+
         try:
-            oid=object._p_oid
-             
             p, serial = self._storage.load(oid, self._version)
 
             # XXX this is quite conservative!


=== Zope/lib/python/ZODB/DB.py 1.39 => 1.40 ===
             conn_no[0] = conn_no[0] + 1
             cn = conn_no[0]
-            for oid, ob in con._cache.items():
+            for oid, ob in con._cache_items():
                 id=''
                 if hasattr(ob,'__dict__'):
                     d=ob.__dict__
@@ -224,10 +224,20 @@
     def cacheSize(self):
         m=[0]
         def f(con, m=m):
-            m[0]=m[0]+len(con._cache)
+            m[0]=m[0]+con._cache.cache_non_ghost_count
 
         self._connectionMap(f)
         return m[0]
+
+    def cacheDetailSize(self):
+        m=[]
+        def f(con, m=m):
+            m.append({'connection':repr(con),
+                      'ngsize':con._cache.cache_non_ghost_count,
+                      'size':len(con._cache)})
+        self._connectionMap(f)
+        m.sort()
+        return m
 
     def close(self): self._storage.close()
 


=== Zope/lib/python/ZODB/cPersistence.c 1.50 => 1.51 ===
 #include "cPersistence.h"
 
+/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
+struct ccobject_head_struct {
+  PyObject_HEAD
+  CPersistentRing ring_home;
+  int non_ghost_count;
+};
+
+#define HOME(O)             ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
+#define NON_GHOST_COUNT(O)  ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
+
 #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
 #define UNLESS(E) if(!(E))
 #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
@@ -112,21 +122,82 @@
 {								 \
   PyObject *r;							 \
       								 \
+  int *count = NON_GHOST_COUNT(self);                            \
+  if(count)                                                      \
+  {                                                              \
+      (*count)++;                                                \
+      self->ring.next = HOME(self);                              \
+      self->ring.prev = HOME(self)->prev;                        \
+      HOME(self)->prev->next = &self->ring;                      \
+      HOME(self)->prev = &self->ring;                            \
+      Py_INCREF(self);                                           \
+  }                                                              \
   self->state=cPersistent_CHANGED_STATE; 	                 \
   UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self))   \
     {                                                            \
-      self->state=cPersistent_GHOST_STATE;                       \
+      ghostify(self);                                            \
       return ER;                                                 \
     }								 \
   self->state=cPersistent_UPTODATE_STATE;			 \
   Py_DECREF(r);							 \
 }
 
+#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
+ if(HOME(self) && self->state>=0) {            \
+ self->ring.prev->next = self->ring.next;      \
+ self->ring.next->prev = self->ring.prev;      \
+ self->ring.next = HOME(self);           \
+ self->ring.prev = HOME(self)->prev;       \
+ HOME(self)->prev->next = &self->ring;    \
+ HOME(self)->prev = &self->ring; }
+
+
 
 /****************************************************************************/
 
 staticforward PyExtensionClass Pertype;
 
+static void
+accessed(cPersistentObject *self)
+{
+    KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
+}
+
+static void
+ghostify(cPersistentObject *self)
+{
+    int *count;
+    count = NON_GHOST_COUNT(self);
+    if(count && (self->state>=0))
+    {
+        (*count)--;
+        self->ring.next->prev = self->ring.prev;
+        self->ring.prev->next = self->ring.next;
+        self->ring.prev = NULL;
+        self->ring.next = NULL;
+        self->state = cPersistent_GHOST_STATE;
+        Py_DECREF(self);
+    }
+    else
+    {
+        self->state = cPersistent_GHOST_STATE;
+    }
+}
+
+static void
+deallocated(cPersistentObject *self)
+{
+    if(self->state>=0) ghostify(self);
+    if(self->cache)
+    {
+        PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
+        if(!v) PyErr_Clear(); /* and explode later */
+        Py_XDECREF(v);
+    }
+    Py_XDECREF(self->jar);
+    Py_XDECREF(self->oid);
+}
+
 static int
 changed(cPersistentObject *self)
 {
@@ -185,7 +256,7 @@
 static PyObject *
 Per__p_deactivate(cPersistentObject *self, PyObject *args)
 {
-  PyObject *dict;
+  PyObject *dict,*dict2=NULL;
 
 #ifdef DEBUG_LOG
   if (idebug_log < 0) call_debug("reinit",self);
@@ -197,13 +268,22 @@
   if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
       HasInstDict(self) && (dict=INSTANCE_DICT(self)))
     {
+      dict2 = PyDict_Copy(dict);
       PyDict_Clear(dict);
       /* Note that we need to set to ghost state unless we are 
 	 called directly. Methods that override this need to
          do the same! */
-      self->state=cPersistent_GHOST_STATE;
+      ghostify(self);
     }
 
+  /* need to delay releasing the last reference on instance attributes
+  until after we have finished accounting for losing our state */
+  if(dict2)
+  {
+      PyDict_Clear(dict2);
+      Py_DECREF(dict2);
+  }
+
   Py_INCREF(Py_None);
   return Py_None;
 }
@@ -333,8 +413,8 @@
 #ifdef DEBUG_LOG
   if(idebug_log < 0) call_debug("del",self);
 #endif
-  Py_XDECREF(self->jar);
-  Py_XDECREF(self->oid);
+  deallocated(self);
+  Py_XDECREF(self->cache);
   Py_DECREF(self->ob_type);
   PyObject_DEL(self);
 }
@@ -387,7 +467,7 @@
 	      {
 		UPDATE_STATE_IF_NECESSARY(self, NULL);
 
-		self->atime=((long)(time(NULL)/3))%65536;
+		KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
 
 		if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
 		    self->serial[5]=='\0' && self->serial[4]=='\0' &&
@@ -419,7 +499,7 @@
     {
       UPDATE_STATE_IF_NECESSARY(self, NULL);
 
-      self->atime=((long)(time(NULL)/3))%65536;
+      KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
     }
 
   return getattrf((PyObject *)self, oname);
@@ -466,6 +546,21 @@
     {
       if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
 	{
+	  if(HOME(self))
+	  {
+	    int result;
+	    if(!v)
+	    {
+              PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
+              return -1;
+	    }
+	    if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
+	    if(result)
+	    {
+              PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
+              return -1;
+            }
+	  }
 	  Py_XINCREF(v);
 	  ASSIGN(self->oid, v);
 	  return 0;
@@ -509,7 +604,6 @@
 	      v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
 	      if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
 	      if (v) { Py_DECREF(v); }
-	      self->state=cPersistent_GHOST_STATE;
 	      return 0;
 	    }
 	  if (PyObject_IsTrue(v)) return changed(self);
@@ -521,8 +615,7 @@
     {
       UPDATE_STATE_IF_NECESSARY(self, -1);
       
-      /* Record access times */
-      self->atime=((long)(time(NULL)/3))%65536;
+      KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
 
       if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
 	 && (self->state != cPersistent_CHANGED_STATE && self->jar)
@@ -680,9 +773,11 @@
   (getattrofunc)Per_getattro,	/*tp_getattr with object key*/
   (setattrofunc)Per_setattro,	/*tp_setattr with object key*/
   changed,
+  accessed,
+  ghostify,
+  deallocated,
   (intfunctionwithpythonarg)Per_setstate,
   (pergetattr)Per_getattr,
-  (persetattr)_setattro,
 };
 
 void


=== Zope/lib/python/ZODB/cPersistence.h 1.21 => 1.22 ===
 #include <time.h>
 
-#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime;  signed char state;  unsigned char reserved; 
+
+#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
 #define cPersistent_GHOST_STATE -1
 #define cPersistent_UPTODATE_STATE 0
 #define cPersistent_CHANGED_STATE 1
 #define cPersistent_STICKY_STATE 2
 
+struct ccobject_head_struct;
+
+typedef struct CPersistentRing_struct
+{
+    struct CPersistentRing_struct *prev;
+    struct CPersistentRing_struct *next;
+} CPersistentRing;
+
 typedef struct {
   cPersistent_HEAD
 } cPersistentObject;
@@ -36,6 +45,9 @@
   getattrofunc getattro;
   setattrofunc setattro;
   int (*changed)(cPersistentObject*);
+  void (*accessed)(cPersistentObject*);
+  void (*ghostify)(cPersistentObject*);
+  void (*deallocated)(cPersistentObject*);
   int (*setstate)(PyObject*);
   pergetattr pergetattro;
   persetattr persetattro;
@@ -59,11 +71,13 @@
 
 #define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
 
-#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE)) 
+#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
+
+#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
 
-#define PER_PREVENT_DEACTIVATION(O)  ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE)) 
+#define PER_PREVENT_DEACTIVATION(O)  ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
 
-#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid);
+#define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
 
 #define PER_USE(O) \
 (((O)->state != cPersistent_GHOST_STATE \
@@ -71,7 +85,7 @@
  ? (((O)->state==cPersistent_UPTODATE_STATE) \
     ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
 
-#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536) 
+#define PER_ACCESSED(O)  (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
 
 #endif
 


=== Zope/lib/python/ZODB/cPickleCache.c 1.41 => 1.42 === (1287/1387 lines abridged)
   Copyright (c) 2001, 2002 Zope Corporation and Contributors.
   All Rights Reserved.
-  
+
   This software is subject to the provisions of the Zope Public License,
   Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
   THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
   WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
   FOR A PARTICULAR PURPOSE
-  
+
  ****************************************************************************/
-static char cPickleCache_doc_string[] = 
+
+static char cPickleCache_doc_string[] =
 "Defines the PickleCache used by ZODB Connection objects.\n"
 "\n"
 "$Id$\n";
 
-/* Compute the current time in the units and range used for peristent
-   objects. */
-#define PER_TIME() ((long)(time(NULL) / 3)) % 65536
+#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
+#define UNLESS(E) if(!(E))
+#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
+#define OBJECT(O) ((PyObject*)O)
 
 #define DONT_USE_CPERSISTENCECAPI
 #include "cPersistence.h"
 #include <time.h>
+#include <stddef.h>
 
 #undef Py_FindMethod
 
-static PyObject *py_reload, *py__p_jar, *py__p_changed;
 
-typedef struct {
-    PyObject_HEAD
-    PyObject *data;
-    PyObject *jar;
-    PyObject *setklassstate;
-    int position;
-    int cache_size;
-    int cache_age;
-    /* Cache statistics */
-    int sum_deal;
-    int sum_deac;
-    double sum_age;
-    int n, na;

[-=- -=- -=- 1287 lines omitted -=- -=- -=-]

+  /* Space for future expansion */
+  0L,0L,0L,0L,
+  ""
 };
 
 static PyObject *
@@ -644,9 +924,9 @@
   int cache_size=100, cache_age=1000;
   PyObject *jar;
 
-  if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
+  UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
       return NULL;
-  return (PyObject *)newccobject(jar, cache_size, cache_age);
+  return (PyObject*)newccobject(jar, cache_size,cache_age);
 }
 
 static struct PyMethodDef cCM_methods[] = {
@@ -657,12 +937,11 @@
 void
 initcPickleCache(void)
 {
-  PyObject *m;
+  PyObject *m, *d;
 
-  Cctype.ob_type = &PyType_Type;
+  Cctype.ob_type=&PyType_Type;
 
-  if (!ExtensionClassImported) 
-      return;
+  UNLESS(ExtensionClassImported) return;
 
   m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
 		     (PyObject*)NULL, PYTHON_API_VERSION);
@@ -670,4 +949,15 @@
   py_reload = PyString_InternFromString("reload");
   py__p_jar = PyString_InternFromString("_p_jar");
   py__p_changed = PyString_InternFromString("_p_changed");
+  py__p_oid = PyString_InternFromString("_p_oid");
+
+  d = PyModule_GetDict(m);
+
+  PyDict_SetItemString(d,"cache_variant",PyString_FromString("stiff/c"));
+
+#ifdef MUCH_RING_CHECKING
+  PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(1));
+#else
+  PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(0));
+#endif
 }