[Zope-Checkins] CVS: Zope2 - ZCatalogIndexes.py:1.2 Catalog.py:1.71 Vocabulary.py:1.14 ZCatalog.py:1.90 __init__.py:1.16
andreas@serenade.digicool.com
andreas@serenade.digicool.com
Wed, 30 May 2001 11:58:08 -0400
Update of /cvs-repository/Zope2/lib/python/Products/ZCatalog
In directory serenade:/tmp/cvs-serv21362/lib/python/Products/ZCatalog
Modified Files:
Catalog.py Vocabulary.py ZCatalog.py __init__.py
Added Files:
ZCatalogIndexes.py
Log Message:
merged ajung-dropin-registry branch
--- Updated File ZCatalogIndexes.py in package Zope2 --
--- Updated File Catalog.py in package Zope2 --
--- Catalog.py 2001/04/27 18:07:13 1.70
+++ Catalog.py 2001/05/30 15:57:37 1.71
@@ -86,8 +86,7 @@
from Persistence import Persistent
import Acquisition
import ExtensionClass
-from SearchIndex import UnIndex, UnTextIndex, UnKeywordIndex
-from SearchIndex.Lexicon import Lexicon
+from Products.PluginIndexes.TextIndex.Lexicon import Lexicon
from MultiMapping import MultiMapping
from string import lower
import Record
@@ -101,7 +100,7 @@
from BTrees.OIBTree import OIBTree
from BTrees.IOBTree import IOBTree
import BTrees.Length
-from SearchIndex.randid import randid
+from Products.PluginIndexes.common.randid import randid
import time
@@ -321,9 +320,14 @@
self.data[key] = tuple(rec)
def addIndex(self, name, index_type):
- """Create a new index, of one of the following index_types
+ """Create a new index, given a name and a index_type.
- Types: 'FieldIndex', 'TextIndex', 'KeywordIndex'.
+ Old format: index_type was a string, 'FieldIndex' 'TextIndex' or
+ 'KeywordIndex' is no longer valid; the actual index must be instantiated
+ and passed in to addIndex.
+
+ New format: index_type is the actual index object to be stored.
+
"""
if self.indexes.has_key(name):
@@ -336,21 +340,12 @@
# pluggable and managable
indexes = self.indexes
- if index_type == 'FieldIndex':
- indexes[name] = UnIndex.UnIndex(name)
- elif index_type == 'TextIndex':
- lexicon=self.lexicon
- if type(lexicon) is type(''):
- lexicon=getattr(self, lexicon).getLexicon()
-
- indexes[name] = UnTextIndex.UnTextIndex(name, None, None, lexicon)
- elif index_type == 'KeywordIndex':
- indexes[name] = UnKeywordIndex.UnKeywordIndex(name)
- else:
- raise 'Unknown Index Type', (
- "%s invalid - must be one of %s"
- % (index_type, ['FieldIndex', 'TextIndex', 'KeywordIndex'])
- )
+
+ if type(index_type) == type(''):
+ raise TypeError,"""Catalog addIndex now requires the index type to
+ be resolved prior to adding; create the proper index in the caller."""
+
+ indexes[name] = index_type;
self.indexes = indexes
@@ -364,9 +359,15 @@
del indexes[name]
self.indexes = indexes
+
+ def reindexIndex(self,name):
+
+ for p in self.paths.items():
+ print p
+
# the cataloging API
- def catalogObject(self, object, uid, threshold=None):
+ def catalogObject(self, object, uid, threshold=None,idxs=[]):
"""
Adds an object to the Catalog by iteratively applying it
all indexes.
@@ -425,7 +426,13 @@
self.paths[index] = uid
total = 0
- for x in self.indexes.values():
+
+ if idxs==[]: use_indexes = self.indexes.keys()
+ else: use_indexes = idxs
+
+ for item in use_indexes:
+ x = self.indexes[item]
+
## tricky! indexes need to acquire now, and because they
## are in a standard dict __getattr__ isn't used, so
## acquisition doesn't kick in, we must explicitly wrap!
@@ -512,25 +519,36 @@
## Searching engine. You don't really have to worry about what goes
## on below here... Most of this stuff came from ZTables with tweaks.
+## But I worry about :-)
- def _indexedSearch(self, args, sort_index, append, used):
+ def _indexedSearch(self, request , sort_index, append, used):
"""
Iterate through the indexes, applying the query to each one.
"""
- rs=None
- data=self.data
+ rs = None # resultset
+ data = self.data
if used is None: used={}
for i in self.indexes.keys():
+
index = self.indexes[i].__of__(self)
if hasattr(index,'_apply_index'):
- r=index._apply_index(args)
+
+ r = None
+
+ # Optimization: we check if there is some work for the index.
+ #
+ if request.has_key(index.id) :
+ if len(request[index.id])>0:
+ r=index._apply_index(request)
+
if r is not None:
r, u = r
- for name in u:
- used[name]=1
+ for name in u: used[name]=1
w, rs = weightedIntersection(rs, r)
+
+
#assert rs==None or hasattr(rs, 'values') or hasattr(rs, 'keys')
if rs is None:
@@ -594,6 +612,7 @@
return used
def searchResults(self, REQUEST=None, used=None, **kw):
+
# Get search arguments:
if REQUEST is None and not kw:
try: REQUEST=self.REQUEST
@@ -634,6 +653,7 @@
# Perform searches with indexes and sort_index
r=[]
+
used=self._indexedSearch(kw, sort_index, r.append, used)
if not r:
return LazyCat(r)
--- Updated File Vocabulary.py in package Zope2 --
--- Vocabulary.py 2001/03/19 19:26:17 1.13
+++ Vocabulary.py 2001/05/30 15:57:37 1.14
@@ -1,203 +1 @@
-##############################################################################
-#
-# Zope Public License (ZPL) Version 1.0
-# -------------------------------------
-#
-# Copyright (c) Digital Creations. All rights reserved.
-#
-# This license has been certified as Open Source(tm).
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# 1. Redistributions in source code must retain the above copyright
-# notice, this list of conditions, and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions, and the following disclaimer in
-# the documentation and/or other materials provided with the
-# distribution.
-#
-# 3. Digital Creations requests that attribution be given to Zope
-# in any manner possible. Zope includes a "Powered by Zope"
-# button that is installed by default. While it is not a license
-# violation to remove this button, it is requested that the
-# attribution remain. A significant investment has been put
-# into Zope, and this effort will continue if the Zope community
-# continues to grow. This is one way to assure that growth.
-#
-# 4. All advertising materials and documentation mentioning
-# features derived from or use of this software must display
-# the following acknowledgement:
-#
-# "This product includes software developed by Digital Creations
-# for use in the Z Object Publishing Environment
-# (http://www.zope.org/)."
-#
-# In the event that the product being advertised includes an
-# intact Zope distribution (with copyright and license included)
-# then this clause is waived.
-#
-# 5. Names associated with Zope or Digital Creations must not be used to
-# endorse or promote products derived from this software without
-# prior written permission from Digital Creations.
-#
-# 6. Modified redistributions of any form whatsoever must retain
-# the following acknowledgment:
-#
-# "This product includes software developed by Digital Creations
-# for use in the Z Object Publishing Environment
-# (http://www.zope.org/)."
-#
-# Intact (re-)distributions of any official Zope release do not
-# require an external acknowledgement.
-#
-# 7. Modifications are encouraged but must be packaged separately as
-# patches to official Zope releases. Distributions that do not
-# clearly separate the patches from the original work must be clearly
-# labeled as unofficial distributions. Modifications which do not
-# carry the name Zope may be packaged in any form, as long as they
-# conform to all of the clauses above.
-#
-#
-# Disclaimer
-#
-# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
-# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
-# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-#
-#
-# This software consists of contributions made by Digital Creations and
-# many individuals on behalf of Digital Creations. Specific
-# attributions are listed in the accompanying credits file.
-#
-##############################################################################
-"""ZCatalog product"""
-
-from Globals import DTMLFile, MessageDialog
-import Globals, AccessControl.Role
-from Acquisition import Implicit
-from Persistence import Persistent
-from OFS.SimpleItem import Item
-from SearchIndex import Lexicon, GlobbingLexicon
-
-from SearchIndex.Lexicon import stop_word_dict
-
-manage_addVocabularyForm=DTMLFile('dtml/addVocabulary',globals())
-
-def manage_addVocabulary(self, id, title, globbing=None, REQUEST=None):
- """Add a Vocabulary object
- """
- id=str(id)
- title=str(title)
- if globbing: globbing=1
-
- c=Vocabulary(id, title, globbing)
- self._setObject(id, c)
- if REQUEST is not None:
- return self.manage_main(self,REQUEST)
-
-
-class Vocabulary(Item, Persistent, Implicit,
- AccessControl.Role.RoleManager,
- ):
- """
- A Vocabulary is a user-managable realization of a Lexicon object.
-
- """
-
- meta_type = "Vocabulary"
- _isAVocabulary = 1
-
-
- manage_options=(
- (
- {'label': 'Vocabulary', 'action': 'manage_main',
- 'help' : ('ZCatalog', 'Vocabulary_Vocabulary.stx')},
- {'label': 'Query', 'action': 'manage_query',
- 'help': ('ZCatalog', 'Vocabulary_Query.stx')},
- )
- +Item.manage_options
- +AccessControl.Role.RoleManager.manage_options
- )
-
- __ac_permissions__=(
-
- ('Manage Vocabulary',
- ['manage_main', 'manage_vocab', 'manage_query'],
- ['Manager']),
-
- ('Query Vocabulary',
- ['query',],
- ['Anonymous', 'Manager']),
- )
-
-
-
- manage_main = DTMLFile('dtml/manage_vocab', globals())
- manage_query = DTMLFile('dtml/vocab_query', globals())
-
- def __init__(self, id, title='', globbing=None):
- """ create the lexicon to manage... """
- self.id = id
- self.title = title
- self.globbing = not not globbing
-
- if globbing:
- self.lexicon = GlobbingLexicon.GlobbingLexicon()
- else:
- self.lexicon = Lexicon.Lexicon(stop_word_dict)
-
- def getLexicon(self):
- return self.lexicon
-
- def query(self, pattern):
- """ """
- result = []
- for x in self.lexicon.get(pattern):
- if self.globbing:
- result.append(self.lexicon._inverseLex[x])
- else:
- result.append(pattern)
- return result
-
-
- def manage_insert(self, word='', URL1=None, RESPONSE=None):
- """ doc string """
- self.insert(word)
-
- if RESPONSE:
- RESPONSE.redirect(URL1 + '/manage_main')
-
- def manage_stop_syn(self, stop_syn, REQUEST=None):
- pass
-
-
-
- def insert(self, word=''):
- self.lexicon.set(word)
-
- def words(self):
- return self.lexicon._lexicon.items()
-
-
-
-
-
-
-
-
-
-
-
-
+from Products.PluginIndexes.TextIndex.Vocabulary import *
--- Updated File ZCatalog.py in package Zope2 --
--- ZCatalog.py 2001/05/21 19:40:55 1.89
+++ ZCatalog.py 2001/05/30 15:57:37 1.90
@@ -86,20 +86,24 @@
from Globals import DTMLFile, MessageDialog
import Globals
+
from OFS.Folder import Folder
from OFS.FindSupport import FindSupport
+from OFS.ObjectManager import ObjectManager
from DateTime import DateTime
-import string, urlparse, urllib, os, sys, time
-import Products
from Acquisition import Implicit
from Persistence import Persistent
from DocumentTemplate.DT_Util import InstanceDict, TemplateDict
from DocumentTemplate.DT_Util import Eval
from AccessControl.Permission import name_trans
from Catalog import Catalog, CatalogError
-from Vocabulary import Vocabulary
from AccessControl import getSecurityManager, full_read_guard
from zLOG import LOG, ERROR
+from ZCatalogIndexes import ZCatalogIndexes
+from Products.PluginIndexes.common.PluggableIndex import PluggableIndexInterface
+from Products.PluginIndexes.TextIndex.Vocabulary import Vocabulary
+from Products.PluginIndexes.TextIndex import Splitter
+import string, urllib, os, sys, time
StringType=type('')
@@ -157,9 +161,8 @@
'action': 'manage_propertiesForm',
'help': ('OFSP','Properties.stx')},
{'label': 'Indexes', # TAB: Indexes
- 'action': 'manage_catalogIndexes',
- 'target': 'manage_main',
- 'help':('ZCatalog','ZCatalog_Indexes.stx')},
+ 'action': 'Indexes/manage_workspace',
+ 'help': ('ZCatalog','ZCatalog_Indexes.stx')},
{'label': 'Metadata', # TAB: Metadata
'action': 'manage_catalogSchema',
'target':'manage_main',
@@ -216,6 +219,8 @@
manage_objectInformation = DTMLFile('dtml/catalogObjectInformation',
globals())
+ Indexes = ZCatalogIndexes()
+
threshold=10000
_v_total=0
_v_transaction = None
@@ -228,6 +233,9 @@
self=self.__of__(container)
self.id=id
self.title=title
+
+ self.vocabulary = None
+ self.availableSplitters = Splitter.availableSplitters
self.threshold = 10000
self._v_total = 0
@@ -235,26 +243,30 @@
if vocab_id is None:
v = Vocabulary('Vocabulary', 'Vocabulary', globbing=1)
self._setObject('Vocabulary', v)
+ self.vocabulary = v
self.vocab_id = 'Vocabulary'
else:
self.vocab_id = vocab_id
+
self._catalog = Catalog(vocabulary=self.vocab_id)
+
+ self.addColumn('id')
+ self.addIndex('id', 'FieldIndex')
- self._catalog.addColumn('id')
- self._catalog.addIndex('id', 'FieldIndex')
+ self.addColumn('title')
+ self.addIndex('title', 'TextIndex')
- self._catalog.addColumn('title')
- self._catalog.addIndex('title', 'TextIndex')
+ self.addColumn('meta_type')
+ self.addIndex('meta_type', 'FieldIndex')
- self._catalog.addColumn('meta_type')
- self._catalog.addIndex('meta_type', 'FieldIndex')
+ self.addColumn('bobobase_modification_time')
+ self.addIndex('bobobase_modification_time', 'FieldIndex')
- self._catalog.addColumn('bobobase_modification_time')
- self._catalog.addIndex('bobobase_modification_time', 'FieldIndex')
+ self.addColumn('summary')
+ self.addIndex('PrincipiaSearchSource', 'TextIndex')
- self._catalog.addColumn('summary')
- self._catalog.addIndex('PrincipiaSearchSource', 'TextIndex')
+ self.addIndex('path','PathIndex')
def __len__(self): return len(self._catalog)
@@ -380,7 +392,7 @@
def manage_addColumn(self, name, REQUEST=None, RESPONSE=None, URL1=None):
""" add a column """
- self._catalog.addColumn(name)
+ self.addColumn(name)
if REQUEST and RESPONSE:
RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Column%20Added')
@@ -388,28 +400,73 @@
def manage_delColumns(self, names, REQUEST=None, RESPONSE=None, URL1=None):
""" del a column """
for name in names:
- self._catalog.delColumn(name)
+ self.delColumn(name)
if REQUEST and RESPONSE:
RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Column%20Deleted')
def manage_addIndex(self, name, type, REQUEST=None, RESPONSE=None, URL1=None):
""" add an index """
- self._catalog.addIndex(name, type)
-
+ self.addIndex(name, type)
+
if REQUEST and RESPONSE:
- RESPONSE.redirect(URL1 + '/manage_catalogIndexes?manage_tabs_message=Index%20Added')
+ RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Index%20Added')
- def manage_delIndexes(self, names, REQUEST=None, RESPONSE=None, URL1=None):
+
+ def manage_deleteIndex(self, ids=None, REQUEST=None, RESPONSE=None,
+ URL1=None):
""" del an index """
- for name in names:
- self._catalog.delIndex(name)
+ if not ids:
+ return MessageDialog(title='No items specified',
+ message='No items were specified!',
+ action = "./manage_main",)
+
+ for name in ids:
+ self.delIndex(name)
if REQUEST and RESPONSE:
- RESPONSE.redirect(URL1 + '/manage_catalogIndexes?manage_tabs_message=Index%20Deleted')
+ RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Index%20Deleted')
+
+ def manage_clearIndex(self, ids=None, REQUEST=None, RESPONSE=None,
+ URL1=None):
+ """ del an index """
+ if not ids:
+ return MessageDialog(title='No items specified',
+ message='No items were specified!',
+ action = "./manage_main",)
+
+ for name in ids:
+ self.clearIndex(name)
+
+ if REQUEST and RESPONSE:
+ RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Index%20Cleared')
+
+
+ def reindexIndex(self,name,REQUEST):
+
+ paths = tuple(self._catalog.paths.values())
+
+ for p in paths:
+ obj = self.resolve_path(p)
+ if not obj:
+ obj = self.resolve_url(p, REQUEST)
+ if obj is not None:
+ self.catalog_object(obj, p, idxs=[name])
+
+ def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None, URL1=None):
+ """ Reindex indexes from a ZCatalog"""
+ if not ids:
+ return MessageDialog(title='No items specified',
+ message='No items were specified!',
+ action = "./manage_main",)
+ for id in ids:
+ self.reindexIndex(id, REQUEST)
- def catalog_object(self, obj, uid=None):
+ if REQUEST and RESPONSE:
+ RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Reindexing%20Performed')
+
+ def catalog_object(self, obj, uid=None, idxs=[]):
""" wrapper around catalog """
if uid is None:
@@ -423,7 +480,7 @@
elif type(uid) is not StringType:
raise CatalogError('The object unique id must be a string.')
- self._catalog.catalogObject(obj, uid, None)
+ self._catalog.catalogObject(obj, uid, None,idxs)
# None passed in to catalogObject as third argument indicates
# that we shouldn't try to commit subtransactions within any
# indexing code. We throw away the result of the call to
@@ -530,13 +587,14 @@
meta_types=() # Sub-object types that are specific to this object
- def all_meta_types(self):
- pmt=()
- if hasattr(self, '_product_meta_types'): pmt=self._product_meta_types
- elif hasattr(self, 'aq_acquire'):
- try: pmt=self.aq_acquire('_product_meta_types')
- except AttributeError: pass
- return self.meta_types+Products.meta_types+pmt
+ # Dont need this anymore -- we inherit from object manager
+ #def all_meta_types(self):
+ # pmt=()
+ # if hasattr(self, '_product_meta_types'): pmt=self._product_meta_types
+ # elif hasattr(self, 'aq_acquire'):
+ # try: pmt=self.aq_acquire('_product_meta_types')
+ # except AttributeError: pass
+ # return self.meta_types+Products.meta_types+pmt
def valid_roles(self):
"Return list of valid roles"
@@ -728,6 +786,54 @@
tt=time.time()-tt
ct=time.clock()-ct
return 'Finished conversion in %s seconds (%s cpu)' % (tt, ct)
+
+ #
+ # Indexing methods
+ #
+
+ def addIndex(self, name, type):
+
+ # Convert the type by finding an appropriate product which supports
+ # this interface by that name. Bleah
+
+ products = ObjectManager.all_meta_types(self, interfaces=(
+ PluggableIndexInterface,))
+
+ p = None
+
+ for prod in products:
+ if prod['name'] == type:
+ p = prod
+ break
+
+ if p is None:
+ raise ValueError, "Index of type %s not found" % type
+
+ base = p['instance']
+
+ if base is None:
+ raise ValueError, "Index type %s does not support addIndex" % type
+
+ index = base(name, self)
+
+ self._catalog.addIndex(name,index)
+
+
+ def delIndex(self, name ):
+
+ self._catalog.delIndex(name)
+
+ def clearIndex(self, name):
+
+ self._catalog.indexes[name].clear()
+
+
+ def addColumn(self, name, default_value=None):
+ return self._catalog.addColumn(name, default_value)
+
+ def delColumn(self, name):
+ return self._catalog.delColumn(name)
+
Globals.default__class_init__(ZCatalog)
@@ -787,6 +893,3 @@
if not (role in pr):
return 0
return 1
-
-
-
--- Updated File __init__.py in package Zope2 --
--- __init__.py 2001/01/31 22:54:11 1.15
+++ __init__.py 2001/05/30 15:57:37 1.16
@@ -85,7 +85,8 @@
"""ZCatalog product"""
-import ZCatalog, Catalog, CatalogAwareness, Vocabulary, ZClasses
+import ZCatalog, Catalog, CatalogAwareness, ZClasses
+from Products.PluginIndexes.TextIndex import Vocabulary
from ZClasses import createZClassForBase
createZClassForBase( ZCatalog.ZCatalog , globals()