[Zope-Checkins] CVS: Zope/lib/python/Products/PluginIndexes/TextIndexNG - DumbQueryParser.py:1.1.2.1 TextIndexNG.py:1.2.2.49 interactiveDemo.py:1.1.2.6
Andreas Jung
andreas@digicool.com
Sat, 2 Mar 2002 09:35:01 -0500
Update of /cvs-repository/Zope/lib/python/Products/PluginIndexes/TextIndexNG
In directory cvs.zope.org:/tmp/cvs-serv23886
Modified Files:
Tag: ajung-textindexng-branch
TextIndexNG.py interactiveDemo.py
Added Files:
Tag: ajung-textindexng-branch
DumbQueryParser.py
Log Message:
added a new dumb query parser to that will be used as a temp, solution
to continue development until the more sophisticated query parser is
finished.
- all unittests will use the "official" query parser
- interactiveDemo.py and the interactive test function in the ZMI
will use the DumpQueryParser
=== Added File Zope/lib/python/Products/PluginIndexes/TextIndexNG/DumbQueryParser.py ===
#!/usr/bin/env python2.1
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""
DumbQueryParser class
A very dumb parser as temporarily solution for the non-yet-finished parser
"""
__version__ = '$Id: DumbQueryParser.py,v 1.1.2.1 2002/03/02 14:35:00 andreasjung Exp $'
from queryparser.Collector import OperatorDict
class DumbQueryParser:
"""Dumb query parser """
def __call__(self, query, operator='and'):
res = []
words = query.split(' ')
words = [ x.strip() for x in words]
for word in words:
if word.startswith('!'):
res.append( 'PL("%s")' % word[1:])
else:
res.append( 'LL("%s")' % word)
s = ','.join(res)
s = "%s(%s)" % (OperatorDict[operator], s)
return s
def test():
import os, sys, re,traceback, atexit
histfile = os.path.expanduser('~/.pyhist')
try:
import readline
readline.read_history_file(histfile)
atexit.register(readline.write_history_file,histfile)
except: pass
print "entering interactive query mode:"
while 1:
s = raw_input('> ')
print s
try:
P = DumbQueryParser()
res = P(s)
print 'res:',res
except:
traceback.print_exc()
if __name__ == '__main__':
test()
=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/TextIndexNG.py 1.2.2.48 => 1.2.2.49 ===
from queryparser.queryparser import QueryParser
+from DumbQueryParser import DumbQueryParser
import Stemmer, Similarity
import Thesaurus, StopWords, Normalizer
@@ -114,11 +115,12 @@
_all_options = ('useSplitter', 'splitterMaxLen', 'splitterIndexNumbers',
'splitterSingleChars', 'splitterCasefolding', 'useStemmer', 'useOperator',
'useGlobbing', 'lexicon', 'nearDistance', 'useSimilarity',
- 'stopWords', 'thesaurus', 'characterMapping', 'useConverters'
+ 'stopWords', 'thesaurus', 'characterMapping', 'useConverters',
+ 'useDumbParser'
)
query_options = ("query","operator")
- _valid_default_operators = ('and','or')
+ _valid_default_operators = ('and','or','near','quote')
def __init__(self
@@ -187,6 +189,9 @@
# use converters from the ConvertersRegistry
self.useConverters = getattr(extra,'useConverters',0)
+ # check Parser
+ self.useDumbParser = getattr(extra,'useDumbParser',0)
+
if verbose: self.debugOn()
else: self.debugOff()
@@ -519,11 +524,7 @@
record = parseIndexRequest(request,self.id,self.query_options)
if record.keys==None: return None
- # Changed for 2.4
- # We use the default operator that can me managed via the ZMI
-
- qop = record.get('operator', self.useOperator)
- query_operator = operator_dict.get(qop)
+ query_operator = record.get('operator', self.useOperator)
if query_operator is None:
raise exceptions.RuntimeError, ("Invalid operator '%s' "
@@ -532,7 +533,7 @@
q = record.keys[0]
- res = self.query( q )
+ res = self.query(q, query_operator )
# this should be fixed
bucket = IIBucket()
@@ -543,8 +544,8 @@
return res[0], (self.id, )
- def query(self, q):
- """ to be finished """
+ def query(self, q, query_operator='and'):
+ """ evaluate query """
LL = self.LexiconLookup
PL = self.SimilarityLookup
@@ -553,9 +554,12 @@
txN = self.txNear
txQ = self.txQuote
- debug('Query: ',q)
+ debug('Query: ',q, query_operator)
- parsed_query = QueryParser()(q)
+ if self.useDumbParser:
+ parsed_query = DumbQueryParser()(q, query_operator)
+ else:
+ parsed_query = QueryParser()(q)
debug('parsed query: ', parsed_query)
@@ -654,7 +658,6 @@
debug("\treturn: ",r)
return r
-
def txUnion(self, *sets):
@@ -947,6 +950,6 @@
def manage_addTextIndexNG(self, id, extra, REQUEST=None, RESPONSE=None, URL3=None):
"""Add a new TextIndexNG """
-
+ extra.useDumbParser = 1
return self.manage_addIndex(id, 'TextIndexNG', extra, REQUEST, RESPONSE, URL3)
=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/interactiveDemo.py 1.1.2.5 => 1.1.2.6 ===
help='max. distance between words for near search',default=5)
+ parser.add_option('-x','--dumbparser', action='store',type='int',
+ dest='extra_useDumbParser',
+ help='enable dumb query parser',default=0)
+
parser.add_option('-c','--casefolding', action='store_true',
dest='extra_splitterCasefolding', help='enable casefolding',
default=1)
@@ -154,7 +158,6 @@
if k.startswith('extra_'):
setattr(E,k[6:],getattr(options,k))
-
cat = index_directory( options.directory,
E,
options.verbose,