[Zope-Checkins] CVS: Zope/lib/python/Products/PluginIndexes/TextIndexNG/queryparser - queryparser.py:1.1.2.12

[Andreas Jung]

Update of /cvs-repository/Zope/lib/python/Products/PluginIndexes/TextIndexNG/queryparser
In directory cvs.zope.org:/tmp/cvs-serv14698

Modified Files:
      Tag: ajung-textindexng-branch
	queryparser.py 
Log Message:
- integrated grammar and terminals definition. 
- code cleanup


=== Zope/lib/python/Products/PluginIndexes/TextIndexNG/queryparser/queryparser.py 1.1.2.11 => 1.1.2.12 ===
 """
 QueryParser class
+
+This module defines the grammar for TextIndexNG queries. It uses the
+kwParsing module. To rebuild the grammer, start this script using the
+'-g' option.
 """
 
 __version__ = '$Id$'
 
-
-import TextIndexG
+import os
 from Collector import Collector
 
+
+if not '__file__' in dir(): # HACK !
+    __file__ = os.getcwd()
+
+COMPILEDFILENAME     = os.path.join(os.path.dirname(__file__), "TextIndexG.py")
+MARSHALLEDFILENAME   = os.path.join(os.path.dirname(__file__), "TextIndexG.mar")
+
+GRAMMARSTRING ="""
+       expr :: 
+
+       @R R1                 ::  expr >> factor
+       @R R2                 ::  expr >> factor or factor
+       @R R3                 ::  expr >> factor factor
+
+       @R R4                 ::  factor >> term
+       @R R5a                ::  factor >> term and term
+       @R R5b                ::  factor >> term near term
+       @R R5c                ::  factor >> term quote term
+
+       @R R6a                ::  term   >> prox_str 
+       @R R6b                ::  term   >> str 
+       @R R7                 ::  term   >> openp expr closep
+"""
+
+Terminals = (
+       ('openp',      '\(',             'addOpenParens'),
+       ('closep',     '\)',             'addCloseParens'),
+       ('and',        'and',            'addOp'),
+       ('near',       'near',           'addOp'),
+       ('quote',      'quote',          'addOp'),
+       ('or',         'or',             'addOp'),
+       ('str',        '[\w\*\?]*',      'addWord'),
+       ('prox_str',   '#[\w\*\?]*',     'addProximityWord'),
+)
+
+
+def DeclareTerminals(Grammar):
+
+    for term, reg, f  in Terminals:
+        Grammar.Addterm(term, reg, '')
+
+def GrammarBuild():
+    """ generatesthe grammar and dump it to a file"""
+
+    import kjParseBuild
+
+    TextIndexG = kjParseBuild.NullCGrammar()
+    TextIndexG.SetCaseSensitivity(0) # grammar is not case sensitive for keywords
+    DeclareTerminals(TextIndexG)
+    TextIndexG.Nonterms("expr term factor ")
+    TextIndexG.Declarerules(GRAMMARSTRING)
+    TextIndexG.Compile()
+
+    print "dumping as python to "+COMPILEDFILENAME
+    outfile = open(COMPILEDFILENAME, "w")
+    TextIndexG.Reconstruct("TextIndexG",outfile,"GRAMMAR")
+    outfile.close()
+
+    print "dumping as binary to "+MARSHALLEDFILENAME
+    outfile = open(MARSHALLEDFILENAME, "w")
+    TextIndexG.MarshalDump(outfile)
+    outfile.close()
+
+    return TextIndexG
+
+try:
+    import TextIndexG
+except:
+    GrammarBuild()
+    import TextIndexG
+
+
 class QueryParser(Collector):
 
     """Query parser takes a TextIndexNG query strings and transforms it
        into a Python expression to be evaluated.
     """ 
 
-    QUOTEREGEX     = 'quote'
-    NEARREGEX      = 'near'
-    ANDREGEX       = 'and'
-    ORREGEX        = 'or'
-    STRREGEX       = '[\w\*\?]*'
-    OPENPREGEX     = '\('
-    CLOSEPREGEX    = '\)'
-
     def __init__(self):
 
         Collector.__init__(self)
@@ -48,13 +115,8 @@
 
     
     def DeclareTerminals(self):
-        self.TextIndexG.Addterm("openp",  self.OPENPREGEX,  self.addOpenParens)
-        self.TextIndexG.Addterm("closep", self.CLOSEPREGEX, self.addCloseParens)
-        self.TextIndexG.Addterm("and",    self.ANDREGEX,    self.addOp)
-        self.TextIndexG.Addterm("near",   self.NEARREGEX,   self.addOp)
-        self.TextIndexG.Addterm("quote",  self.QUOTEREGEX,  self.addOp)
-        self.TextIndexG.Addterm("or",     self.ORREGEX,     self.addOp)
-        self.TextIndexG.Addterm("str",    self.STRREGEX,    self.addWord)
+        for term, reg, f  in Terminals:
+            self.TextIndexG.Addterm(term, reg, getattr(self,f) )
 
 
     def BindRules(self): pass
@@ -70,33 +132,42 @@
 def test():
 
     import os, sys, re,traceback, atexit
-    import readline
 
     histfile = os.path.expanduser('~/.pyhist')
     try:
+        import readline
         readline.read_history_file(histfile)
-    except IOError: pass
-    atexit.register(readline.write_history_file,histfile)
-
-    print "loading grammar as python"
+        atexit.register(readline.write_history_file,histfile)
+    except: pass
 
     Context = { }
 
-
+    print "entering interactive query mode:"
     while 1:
 
         s = raw_input('> ')
-
         print s  
 
         try:
+
             P = QueryParser()
             res = P(s)        
-
             print 'res:',res
+
         except:
             traceback.print_exc()
 
+
 if __name__ == '__main__':
-    test()
+
+    import sys
+
+    if len(sys.argv)>1 and sys.argv[1]=='-g':
+
+        print "(re)generating the TextIndexG grammar in file TextIndexG.py"
+        Dummy = GrammarBuild()
+        print "(re)generation done."
+
+    else:
+        test()