[Zope-Checkins] CVS: Zope/lib/python/Products/PluginIndexes/TextIndexNG - TextIndexNG.py:1.2.2.56 LexiconNG.py:1.1.2.6

[Andreas Jung]

Update of /cvs-repository/Zope/lib/python/Products/PluginIndexes/TextIndexNG
In directory cvs.zope.org:/tmp/cvs-serv8930

Modified Files:
      Tag: ajung-textindexng-branch
	TextIndexNG.py LexiconNG.py 
Log Message:
- replaced main loop inside the vocabulary objects to assign
  wordIds by C implementation (50% speedup)
- LexiconNG code cleanup


=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/TextIndexNG.py 1.2.2.55 => 1.2.2.56 ===
 
         self._v_getWordIdList  = self._lexicon.getWordIdList
-        self._v_getWordId      = self._lexicon.getWordId
-        self._v_getWordById    = self._lexicon.getWord
-        self._v_getIdByWord    = self._lexicon.get
 
 
     def __nonzero__(self):
@@ -499,7 +496,7 @@
             debug('\tCasefolding: ',word)
 
         # Lookup list of wordIds (usually should contain only *one*)
-        wids = self._v_getIdByWord(word)
+        wids = self._lexicon.get(word)
         debug("\tWids: ", wids)
 
         # Retrieve list of docIds for that wordId 


=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/LexiconNG.py 1.1.2.5 => 1.1.2.6 ===
 from Products.PluginIndexes.TextIndex.randid import randid
 from types import StringType
+from indexsupport import vocabBatchInsert
 
 
 class LexiconNG(Persistent, Implicit):
@@ -50,56 +51,13 @@
     def getWordIdList(self,words):
         """ return a list a wordIds for a list of words """
 
-        lst = []
-        for word in words:
+        return vocabBatchInsert(
+                    self._lexicon, 
+                    self._inverseLex, 
+                    words
+                )
 
-            wid=self._lexicon.get(word, None)
-
-            if wid is None: 
-                wid=self.assignWordId(word)
-            lst.append(wid)
-
-        return lst
-
-        
-
-    def getWordId(self, word):
-        """ return the word id of 'word' """
-
-        wid=self._lexicon.get(word, None)
-
-        if wid is None: 
-            wid=self.assignWordId(word)
-        return wid
-        
-    set = getWordId
-
-    def getWord(self, wid):
-        """ post-2.3.1b2 method, will not work with unconverted lexicons """
-        return self._inverseLex.get(wid, None)
-        
-
-    def assignWordId(self, word):
-        """Assigns a new word id to the provided word and returns it."""
-
-        # First make sure it's not already in there
-
-        if self._lexicon.has_key(word):
-            return self._lexicon[word]
-
-        wid=self.num + 1
-        while not self._inverseLex.insert(wid, word):
-            wid=randid()
-    
-        # we removed some optimizations here
-        self._lexicon[word] = wid
-
-        self.num+=1            
-
-        return wid
-
-
-    def get(self, key, default=None):
+    def getWordId(self, key, default=None):
         """Return the matched word against the key."""
 
         r=IISet()
@@ -109,10 +67,12 @@
 
         return r
 
+    get = getWordId
+
+
     def __getitem__(self, key):
         return self.get(key)
 
 
     def __len__(self):
         return len(self._lexicon)
-