[Zope-Checkins] SVN: Zope/trunk/ merge ajung-zcatalog-progress
branch 26525:26606
Andreas Jung
andreas at andreas-jung.com
Sat Jul 17 00:16:35 EDT 2004
Log message for revision 26607:
merge ajung-zcatalog-progress branch 26525:26606
Changed:
U Zope/trunk/doc/CHANGES.txt
U Zope/trunk/lib/python/Products/ZCatalog/Catalog.py
U Zope/trunk/lib/python/Products/ZCatalog/CatalogPathAwareness.py
A Zope/trunk/lib/python/Products/ZCatalog/ProgressHandler.py
U Zope/trunk/lib/python/Products/ZCatalog/README.txt
U Zope/trunk/lib/python/Products/ZCatalog/ZCatalog.py
U Zope/trunk/lib/python/Products/ZCatalog/ZCatalogIndexes.py
U Zope/trunk/lib/python/Products/ZCatalog/__init__.py
U Zope/trunk/lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml
-=-
Modified: Zope/trunk/doc/CHANGES.txt
===================================================================
--- Zope/trunk/doc/CHANGES.txt 2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/doc/CHANGES.txt 2004-07-17 04:16:35 UTC (rev 26607)
@@ -12,10 +12,6 @@
changes in C, brining the Python and C implementations back in
sync. See lib/python/AccessControl/PermissionRole.py.
- - Renable C Zope security policy by implementing recent Python
- changes in C, brining the Python and C implementations back in
- sync. See lib/python/AccessControl/ZopeSecurityPolicy.py.
-
- Add cyclic-garbage collection support to C extension classes,
especially to acquisition wrappers.
@@ -26,6 +22,10 @@
Features added
+ - ZCatalog: added a new configuration option in the "Advanced" tab
+ to provide optional logging of the progress of long running
+ reindexing or recataloging operations.
+
- made Zope.configure return the starter instance to enable other
methods to be called, such as starter.setupConfiguredLoggers()
Modified: Zope/trunk/lib/python/Products/ZCatalog/Catalog.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/Catalog.py 2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/Catalog.py 2004-07-17 04:16:35 UTC (rev 26607)
@@ -11,27 +11,27 @@
#
##############################################################################
-from Persistence import Persistent
+import types
+import logging
+from bisect import bisect
+from random import randint
+
import Acquisition
import ExtensionClass
-from MultiMapping import MultiMapping
-import Record
from Missing import MV
-import logging
+from Persistence import Persistent
-from Lazy import LazyMap, LazyFilter, LazyCat, LazyValues
-from CatalogBrains import AbstractCatalogBrain, NoBrainer
+import BTrees.Length
from BTrees.IIBTree import intersection, weightedIntersection, IISet
from BTrees.OIBTree import OIBTree
from BTrees.IOBTree import IOBTree
-import BTrees.Length
+from Lazy import LazyMap, LazyCat, LazyValues
+from CatalogBrains import AbstractCatalogBrain, NoBrainer
-import time, sys, types
-from bisect import bisect
-from random import randint
LOG = logging.getLogger('Zope.ZCatalog')
+
try:
from DocumentTemplate.cDocumentTemplate import safe_callable
except ImportError:
@@ -278,7 +278,7 @@
indexes = self.indexes
- if isinstance(index_type, types.StringType):
+ if isinstance(index_type, str):
raise TypeError,"""Catalog addIndex now requires the index type to
be resolved prior to adding; create the proper index in the caller."""
@@ -755,7 +755,7 @@
reverse = 0
if sort_index is not None:
order = self._get_sort_attr("order", args)
- if (isinstance(order, types.StringType) and
+ if (isinstance(order, str) and
order.lower() in ('reverse', 'descending')):
reverse = 1
# Perform searches with indexes and sort_index
Modified: Zope/trunk/lib/python/Products/ZCatalog/CatalogPathAwareness.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/CatalogPathAwareness.py 2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/CatalogPathAwareness.py 2004-07-17 04:16:35 UTC (rev 26607)
@@ -13,7 +13,6 @@
"""ZCatalog Findable class"""
-import urllib
from Globals import DTMLFile
from Acquisition import aq_base
Copied: Zope/trunk/lib/python/Products/ZCatalog/ProgressHandler.py (from rev 26606, Zope/branches/ajung-zcatalog-progress/lib/python/Products/ZCatalog/ProgressHandler.py)
Property changes on: Zope/trunk/lib/python/Products/ZCatalog/ProgressHandler.py
___________________________________________________________________
Name: svn:keywords
+ svn:eol-style
Modified: Zope/trunk/lib/python/Products/ZCatalog/README.txt
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/README.txt 2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/README.txt 2004-07-17 04:16:35 UTC (rev 26607)
@@ -52,3 +52,11 @@
encoding (set in site.py of the Python installation) if any
to the keyword uses a non-ascii encoding (e.g. using accented
characters).
+
+ Notes for Zope 2.8:
+
+ reindexIndex() and refreshCatalog() accept a new optional parameter
+ 'pghandler' which must be a handler instance implementing the
+ IProgressHandler interface (see ProgressHandler.py). This can be useful
+ to provide logging during long running operations.
+
Modified: Zope/trunk/lib/python/Products/ZCatalog/ZCatalog.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/ZCatalog.py 2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/ZCatalog.py 2004-07-17 04:16:35 UTC (rev 26607)
@@ -15,6 +15,8 @@
$Id$
"""
+import urllib, time, sys, string,logging
+
from Globals import DTMLFile, MessageDialog
import Globals
@@ -35,9 +37,8 @@
from Products.PluginIndexes.common.PluggableIndex \
import PluggableIndexInterface
from Products.PluginIndexes.TextIndex import Splitter
-import urllib, time, sys
-import string,logging
from IZCatalog import IZCatalog
+from ProgressHandler import ZLogHandler
LOG = logging.getLogger('Zope.ZCatalog')
@@ -127,6 +128,7 @@
'manage_catalogClear', 'manage_addColumn', 'manage_delColumn',
'manage_addIndex', 'manage_delIndex', 'manage_clearIndex',
'manage_reindexIndex', 'manage_main', 'availableSplitters',
+ 'manage_setProgress',
# these two are deprecated:
'manage_delColumns', 'manage_deleteIndex'
@@ -250,7 +252,9 @@
elapse = time.time()
c_elapse = time.clock()
- self.refreshCatalog(clear=1)
+ pgthreshold = self._getProgressThreshold()
+ handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
+ self.refreshCatalog(clear=1, pghandler=handler)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
@@ -263,7 +267,7 @@
'Total CPU time: %s' % (`elapse`, `c_elapse`)))
- def refreshCatalog(self, clear=0):
+ def refreshCatalog(self, clear=0, pghandler=None):
""" re-index everything we can find """
cat = self._catalog
@@ -272,26 +276,27 @@
paths = tuple(paths)
cat.clear()
- LOG('ZCatalog', BLATHER, 'Starting recataloging of ZCatalog at %s' %
- self.absolute_url(1))
num_objects = len(paths)
+ if pghandler:
+ pghandler.init('Refreshing catalog: %s' % self.absolute_url(1), num_objects)
+
for i in xrange(num_objects):
+ if pghandler: pghandler.report(i)
+
p = paths[i]
obj = self.resolve_path(p)
if not obj:
obj = self.resolve_url(p, self.REQUEST)
if obj is not None:
try:
- LOG('ZCatalog', BLATHER, 'Recataloging object %s (%d/%d)' %
- (p, i, num_objects))
- self.catalog_object(obj, p)
+ self.catalog_object(obj, p, pghandler=pghandler)
except ConflictError:
raise
except:
- LOG('ZCatalog', ERROR, 'Recataloging object at %s failed' % p,
- error=sys.exc_info())
+ LOG.error('Recataloging object at %s failed' % p,
+ exc_info=sys.exc_info())
- LOG('ZCatalog', BLATHER, 'Recataloging of ZCatalog at %s terminated' % self.absolute_url(1))
+ if pghandler: pghandler.finish()
def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None):
""" clears the whole enchilada """
@@ -460,10 +465,21 @@
'/manage_catalogIndexes?manage_tabs_message=Index%20Cleared')
- def reindexIndex(self, name, REQUEST):
+ def reindexIndex(self, name, REQUEST, pghandler=None):
if isinstance(name, str):
name = (name,)
- for p in self._catalog.uids.keys():
+
+ paths = self._catalog.uids.keys()
+ num_paths = len(paths) # inefficient
+
+ i = 0
+ if pghandler:
+ pghandler.init('reindexing %s' % name, num_paths)
+
+ for p in paths:
+ i+=1
+ if pghandler: pghandler.report(i)
+
obj = self.resolve_path(p)
if not obj:
obj = self.resolve_url(p, REQUEST)
@@ -475,7 +491,7 @@
# index via the UI
try:
self.catalog_object(obj, p, idxs=name,
- update_metadata=0)
+ update_metadata=0, pghandler=pghandler)
except TypeError:
# Fall back to Zope 2.6.2 interface. This is necessary for
# products like CMF 1.4.2 and earlier that subclass from
@@ -485,8 +501,11 @@
warn('catalog_object interface of %s not up to date'
% self.__class__.__name__,
DeprecationWarning)
- self.catalog_object(obj, p, idxs=name)
+ self.catalog_object(obj, p, idxs=name, pghandler=pghandler)
+ if pghandler:
+ pghandler.finish()
+
def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None,
URL1=None):
"""Reindex indexe(s) from a ZCatalog"""
@@ -495,7 +514,9 @@
message='No items were specified!',
action = "./manage_catalogIndexes",)
- self.reindexIndex(ids, REQUEST)
+ pgthreshold = self._getProgressThreshold()
+ handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
+ self.reindexIndex(ids, REQUEST, handler)
if REQUEST and RESPONSE:
RESPONSE.redirect(
@@ -509,7 +530,7 @@
return Splitter.availableSplitters
- def catalog_object(self, obj, uid=None, idxs=None, update_metadata=1):
+ def catalog_object(self, obj, uid=None, idxs=None, update_metadata=1, pghandler=None):
""" wrapper around catalog """
if uid is None:
@@ -551,6 +572,8 @@
get_transaction().commit(1)
self._p_jar.cacheGC()
self._v_total = 0
+ if pghandler:
+ pghandler.info('commiting subtransaction')
def uncatalog_object(self, uid):
"""Wrapper around catalog """
@@ -861,6 +884,21 @@
'%s unchanged.' % (len(fixed), len(removed), unchanged),
action='./manage_main')
+ def manage_setProgress(self, pgthreshold=0, RESPONSE=None, URL1=None):
+ """Set parameter to perform logging of reindexing operations very
+ 'pgthreshold' objects
+ """
+
+ self.pgthreshold = pgthreshold
+ if RESPONSE:
+ RESPONSE.redirect(
+ URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed')
+
+ def _getProgressThreshold(self):
+ if not hasattr(self, 'pgthreshold'):
+ self.pgthreshold = 0
+ return self.pgthreshold
+
def manage_convertBTrees(self, threshold=200):
"""Convert the catalog's data structures to use BTrees package"""
assert type(threshold) is type(0)
Modified: Zope/trunk/lib/python/Products/ZCatalog/ZCatalogIndexes.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/ZCatalogIndexes.py 2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/ZCatalogIndexes.py 2004-07-17 04:16:35 UTC (rev 26607)
@@ -14,21 +14,16 @@
"""$Id$
"""
-from Globals import DTMLFile, InitializeClass
+
+from Acquisition import Implicit
+from Persistence import Persistent
+from Globals import DTMLFile, InitializeClass
from AccessControl.SecurityInfo import ClassSecurityInfo
-from AccessControl.Permissions import delete_objects, manage_zcatalog_indexes
-import Globals
+from AccessControl.Permissions import manage_zcatalog_indexes
from OFS.Folder import Folder
-from OFS.FindSupport import FindSupport
-from OFS.History import Historical
from OFS.SimpleItem import SimpleItem
-from OFS.ObjectManager import ObjectManager, IFAwareObjectManager
+from OFS.ObjectManager import IFAwareObjectManager
-import os, sys, time
-
-from Acquisition import Implicit
-from Persistence import Persistent
-
from Products.PluginIndexes.common.PluggableIndex import PluggableIndexInterface
_marker = []
Modified: Zope/trunk/lib/python/Products/ZCatalog/__init__.py
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/__init__.py 2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/__init__.py 2004-07-17 04:16:35 UTC (rev 26607)
@@ -13,7 +13,7 @@
"""ZCatalog product"""
-import ZCatalog, Catalog, CatalogAwareness, CatalogPathAwareness, ZClasses
+import ZCatalog, CatalogAwareness, CatalogPathAwareness
from Products.PluginIndexes.TextIndex import Vocabulary
from ZClasses import createZClassForBase
Modified: Zope/trunk/lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml
===================================================================
--- Zope/trunk/lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml 2004-07-16 22:11:49 UTC (rev 26606)
+++ Zope/trunk/lib/python/Products/ZCatalog/dtml/catalogAdvanced.dtml 2004-07-17 04:16:35 UTC (rev 26607)
@@ -37,6 +37,19 @@
</td>
</tr>
<tr>
+ <td align="left" valign="top">
+ <p class="form-help">Log progress of reindexing every N objects to the Zope logger (set to 0 to disabled logging)
+ </p>
+ </td>
+ <td align="right" valign="top">
+<form action="&dtml-URL1;">
+<input type="text" name="pgthreshold:int" value="<dtml-var pgthreshold missing="0">">
+<input class="form-element" type="submit"
+ name="manage_setProgress:method" value=" Change ">
+</form>
+ </td>
+</tr>
+<tr>
<td>
</td>
More information about the Zope-Checkins
mailing list