[Zope-Checkins] CVS: Zope/lib/python/Products/PluginIndexes/TextIndex - GlobbingLexicon.py:1.13 Lexicon.py:1.7 Vocabulary.py:1.9

[Andreas Jung]

Update of /cvs-repository/Zope/lib/python/Products/PluginIndexes/TextIndex
In directory cvs.zope.org:/tmp/cvs-serv11212/lib/python/Products/PluginIndexes/TextIndex

Modified Files:
	GlobbingLexicon.py Lexicon.py Vocabulary.py 
Log Message:
      - TextIndex: Enhanced splitter functionality now allows the
        TextIndex to index numbers, single characters. It is also 
        possible to enable case-sensitive indexing. The new 
        configuration options are available through the addForm
        of the Vocabulary object.
        


=== Zope/lib/python/Products/PluginIndexes/TextIndex/GlobbingLexicon.py 1.12 => 1.13 ===
 import Splitter
 
-
 import re, string
 
 from BTrees.IIBTree import IISet, union, IITreeSet
@@ -56,9 +55,10 @@
     eow = '$'
 
 
-    def __init__(self,useSplitter=None):
+    def __init__(self,useSplitter=None,extra=None):
         self.clear()
         self.useSplitter = useSplitter
+        self.splitterParams = extra
         self.SplitterFunc = Splitter.getSplitter(self.useSplitter)
 
     def clear(self):
@@ -239,9 +239,16 @@
         ## sense in stemming a globbing lexicon.
 
         try:
-            return self.SplitterFunc(astring,None,encoding)
+            return self.SplitterFunc(
+                    astring, 
+                    words, 
+                    encoding=encoding,
+                    singlechar=self.splitterParams.splitterSingleChars,
+                    indexnumbers=self.splitterParams.splitterIndexNumbers,
+                    casefolding=self.splitterParams.splitterCasefolding
+                    )
         except:
-            return self.SplitterFunc(astring,None)
+            return self.SplitterFunc(astring, words)
 
 
     def createRegex(self, pat):
@@ -268,5 +275,4 @@
         result = result.replace( '?', '.')
 
         return "%s$" % result 
-
 


=== Zope/lib/python/Products/PluginIndexes/TextIndex/Lexicon.py 1.6 => 1.7 ===
     stop_syn={}
 
-    def __init__(self, stop_syn=None,useSplitter=None):
-
+    def __init__(self, stop_syn=None,useSplitter=None,extra=None):
 
         self.clear()
         if stop_syn is None:
@@ -52,7 +51,7 @@
     
         self.useSplitter = Splitter.splitterNames[0]
         if useSplitter: self.useSplitter=useSplitter
-
+        self.splitterParams = extra
         self.SplitterFunc = Splitter.getSplitter(self.useSplitter)
 
 
@@ -153,10 +152,17 @@
 
     def Splitter(self, astring, words=None, encoding = "latin1"):
         """ wrap the splitter """
-        if words is None:
-            words = self.stop_syn
+        if words is None: words = self.stop_syn
+
         try:
-            return self.SplitterFunc(astring, words, encoding)
+            return self.SplitterFunc(
+                    astring, 
+                    words, 
+                    encoding=encoding,
+                    singlechar=self.splitterParams.splitterSingleChars,
+                    indexnumbers=self.splitterParams.splitterIndexNumbers,
+                    casefolding=self.splitterParams.splitterCasefolding
+                    )
         except:
             return self.SplitterFunc(astring, words)
 
@@ -164,10 +170,6 @@
     def query_hook(self, q):
         """ we don't want to modify the query cuz we're dumb """
         return q
-        
-
-
-
 
 stop_words=(
     'am', 'ii', 'iii', 'per', 'po', 're', 'a', 'about', 'above', 'across',
@@ -216,7 +218,4 @@
     )
 stop_word_dict={}
 for word in stop_words: stop_word_dict[word]=None
-
-
-
 


=== Zope/lib/python/Products/PluginIndexes/TextIndex/Vocabulary.py 1.8 => 1.9 ===
 manage_addVocabularyForm=DTMLFile('dtml/addVocabulary',globals())
 
-def manage_addVocabulary(self, id, title, globbing=None, splitter='', REQUEST=None):
+def manage_addVocabulary(self, id, title, globbing=None, extra=None,
+                         splitter='', REQUEST=None):
     """Add a Vocabulary object
     """
     id=str(id)
     title=str(title)
     if globbing: globbing=1
-    
-    c=Vocabulary(id, title, globbing,splitter)
+
+    c=Vocabulary(id, title, globbing,splitter,extra)
     self._setObject(id, c)
     if REQUEST is not None:
         return self.manage_main(self,REQUEST,update_menu=1)
 
+class _extra: pass
+
 
 class Vocabulary(Item, Persistent, Implicit,
                  AccessControl.Role.RoleManager,
@@ -75,20 +78,28 @@
     manage_main = DTMLFile('dtml/manage_vocab', globals())
     manage_query = DTMLFile('dtml/vocab_query', globals())
 
-    def __init__(self, id, title='', globbing=None,splitter=None):
+    def __init__(self, id, title='', globbing=None,splitter=None,extra=None):
         """ create the lexicon to manage... """
         self.id = id
         self.title = title
         self.globbing = not not globbing
-            
+
         self.useSplitter = Splitter.splitterNames[0]    
         if splitter:
             self.useSplitter = splitter
 
+        if not extra:
+            extra = _extra()
+            extra.splitterIndexNumbers = 0
+            extra.splitterSingleChars  = 0
+            extra.splitterCasefolding  = 1
+
         if globbing:
-            self.lexicon = GlobbingLexicon.GlobbingLexicon(useSplitter=self.useSplitter)
+            self.lexicon = GlobbingLexicon.GlobbingLexicon(
+                                useSplitter=self.useSplitter,extra=extra)
         else:
-            self.lexicon = Lexicon.Lexicon(stop_word_dict,useSplitter=self.useSplitter)
+            self.lexicon = Lexicon.Lexicon(stop_word_dict,
+                                useSplitter=self.useSplitter,extra=extra)
 
     def getLexicon(self):
         return self.lexicon
@@ -114,8 +125,6 @@
 
     def manage_stop_syn(self, stop_syn, REQUEST=None):
         pass
-
-
 
     def insert(self, word=''):
         self.lexicon.set(word)