[Zope-Checkins] SVN: Zope/trunk/src/Products/ZCTextIndex/ Merged Zope/branches/tyam-unicodeSplitterPatch 104723:104761

[Takeshi Yamamoto]

Log message for revision 105051:
  Merged Zope/branches/tyam-unicodeSplitterPatch 104723:104761

Changed:
  U   Zope/trunk/src/Products/ZCTextIndex/Lexicon.py
  U   Zope/trunk/src/Products/ZCTextIndex/tests/testLexicon.py

-=-
Modified: Zope/trunk/src/Products/ZCTextIndex/Lexicon.py
===================================================================
--- Zope/trunk/src/Products/ZCTextIndex/Lexicon.py	2009-10-13 16:05:59 UTC (rev 105050)
+++ Zope/trunk/src/Products/ZCTextIndex/Lexicon.py	2009-10-13 16:28:22 UTC (rev 105051)
@@ -76,7 +76,8 @@
     def termToWordIds(self, text):
         last = _text2list(text)
         for element in self._pipeline:
-            last = element.process(last)
+            process = getattr(element, "process_post_glob", element.process) 
+            last = process(last)
         wids = []
         for word in last:
             wids.append(self._wids.get(word, 0))

Modified: Zope/trunk/src/Products/ZCTextIndex/tests/testLexicon.py
===================================================================
--- Zope/trunk/src/Products/ZCTextIndex/tests/testLexicon.py	2009-10-13 16:05:59 UTC (rev 105050)
+++ Zope/trunk/src/Products/ZCTextIndex/tests/testLexicon.py	2009-10-13 16:28:22 UTC (rev 105051)
@@ -94,6 +94,28 @@
         wids = lexicon.termToWordIds('boxes')
         self.assertEqual(wids, [0])
 
+    def testTermToWordIdsWithProcess_post_glob(self):
+        """This test is for added process_post_glob"""
+        class AddedSplitter(Splitter):
+            def process_post_glob(self, lst):
+                assert lst == ['dogs']
+                return ['dogs']
+        lexicon = Lexicon(AddedSplitter())
+        wids = lexicon.sourceToWordIds('cats and dogs')
+        wids = lexicon.termToWordIds('dogs')
+        self.assertEqual(wids, [3])
+
+    def testMissingTermToWordIdsWithProcess_post_glob(self):
+        """This test is for added process_post_glob"""
+        class AddedSplitter(Splitter):
+            def process_post_glob(self, lst):
+                assert lst == ['dogs']
+                return ['fox']
+        lexicon = Lexicon(AddedSplitter())
+        wids = lexicon.sourceToWordIds('cats and dogs')
+        wids = lexicon.termToWordIds('dogs')
+        self.assertEqual(wids, [0])
+
     def testOnePipelineElement(self):
         lexicon = Lexicon(Splitter(), StupidPipelineElement('dogs', 'fish'))
         wids = lexicon.sourceToWordIds('cats and dogs')