[Zope-Checkins] CVS: Zope/lib/python/Products/PluginIndexes/TextIndexNG - BaseProximityLexicon.py:1.1.2.6

[Andreas Jung]

Update of /cvs-repository/Zope/lib/python/Products/PluginIndexes/TextIndexNG
In directory cvs.zope.org:/tmp/cvs-serv25555

Modified Files:
      Tag: ajung-textindexng-branch
	BaseProximityLexicon.py 
Log Message:
ProximityLexicon is now fully unicode aware. The proximity algorithm
is applied against a transformed ASCII version of the string. 



=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/BaseProximityLexicon.py 1.1.2.5 => 1.1.2.6 ===
 
 from Products.PluginIndexes.TextIndex.randid import randid
-from types import StringType
+from types import StringType, UnicodeType
 
 import Proximity
 
@@ -60,7 +60,13 @@
     def getWordId(self, word):
         """ return the word id of 'word' """
 
-        word = self._v_proximity(word)
+        try:
+            word = self._v_proximity(word)
+        except TypeError:
+            if isinstance(word, UnicodeType):
+                word = ''.join([ chr(ord(x) & 127) for x in word])
+                word = self._v_proximity(word)
+ 
 
         wid=self._lexicon.get(word, None)
 
@@ -100,13 +106,19 @@
         return wid
 
 
-    def get(self, key, default=None):
+    def get(self, word, default=None):
         """Return the matched word against the key."""
 
-        key = self._v_proximity(key)
+        try:
+            word = self._v_proximity(word)
+        except TypeError:
+            if isinstance(word, UnicodeType):
+                word = ''.join([ chr(ord(x) & 127) for x in word])
+                word = self._v_proximity(word)
+
 
         r=IISet()
-        wid=self._lexicon.get(key, default)
+        wid=self._lexicon.get(word, default)
         if wid is not None: r.insert(wid)
         return r