[Zope-Checkins] CVS: Zope/lib/python/Products/PluginIndexes/TextIndexNG - TextIndexNG.py:1.2.2.47

[Andreas Jung]

Update of /cvs-repository/Zope/lib/python/Products/PluginIndexes/TextIndexNG
In directory cvs.zope.org:/tmp/cvs-serv18318

Modified Files:
      Tag: ajung-textindexng-branch
	TextIndexNG.py 
Log Message:
full integration of converters into TextIndexNG


=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/TextIndexNG.py 1.2.2.46 => 1.2.2.47 ===
 from Products.PluginIndexes import PluggableIndex       
 from Products.PluginIndexes.common.util import parseIndexRequest
-
+from OFS.content_types import guess_content_type
 from BTrees.IOBTree import IOBTree
 from BTrees.OOBTree import OOBTree
 from BTrees.IIBTree import IIBTree, IIBucket, IISet
@@ -111,7 +111,7 @@
     _all_options = ('useSplitter', 'splitterMaxLen', 'splitterIndexNumbers',
          'splitterSingleChars', 'splitterCasefolding', 'useStemmer', 'useOperator',
          'useGlobbing', 'lexicon', 'nearDistance', 'useSimilarity',
-         'stopWords', 'thesaurus', 'characterMapping', 'useConverter'
+         'stopWords', 'thesaurus', 'characterMapping', 'useConverters'
         )
 
     query_options = ("query","operator")
@@ -181,9 +181,8 @@
         # Normalizer: characterMapping
         self.charMapping   = getattr(extra,'characterMapping', None) or None
 
-
         # use converters from the ConvertersRegistry
-        self.useConverter = getattr(extra,'useConverter'), 0)
+        self.useConverters = getattr(extra,'useConverters',0) 
 
         if verbose: self.debugOn()
         else:       self.debugOff()
@@ -394,9 +393,18 @@
         except (AttributeError, TypeError):
             encoding = 'latin1'
 
-
         T("encoding")
 
+        # Document converter
+
+        if self.useConverters:
+            mimetype = guess_content_type(source, obj.getId())
+            
+            converter = ConverterRegistry.getConverterFor( mimetype )
+            if converter:
+                source = converter(source)
+
+        T('converter')
 
         # Split the text into a list of words