[Zope3-checkins] CVS: Zope3/src/zope/app/rdb/gadfly - COPYRIGHT:1.2 DLispShort.py:1.2 DumbLispGen.py:1.2 SQLTESTG.py:1.2 __init__.py:1.2 gadfly.html:1.2 gadfly.py:1.2 gfSQL.html:1.2 gfclient.py:1.2 gfdb0.py:1.2 gffaq.html:1.2 gfinstall.py:1.2 gfintrospect.py:1.2 gfrecover.html:1.2 gfserve.py:1.2 gfsocket.py:1.2 gfstest.py:1.2 gftest.py:1.2 idl.py:1.2 index.html:1.2 kjParseBuild.py:1.2 kjParser.py:1.2 kjSet.py:1.2 kjbuckets0.py:1.2 kjpylint.py:1.2 kwParsing.html:1.2 pygram.py:1.2 relalg.py:1.2 remotetest.py:1.2 server.html:1.2 sql.mar:1.2 sqlbind.py:1.2 sqlgen.py:1.2 sqlgram.py:1.2 sqlgtest.py:1.2 sqlmod.py:1.2 sqlsem.py:1.2 sqlwhere.py:1.2
Jim Fulton
jim@zope.com
Wed, 25 Dec 2002 09:13:48 -0500
Update of /cvs-repository/Zope3/src/zope/app/rdb/gadfly
In directory cvs.zope.org:/tmp/cvs-serv15352/src/zope/app/rdb/gadfly
Added Files:
COPYRIGHT DLispShort.py DumbLispGen.py SQLTESTG.py __init__.py
gadfly.html gadfly.py gfSQL.html gfclient.py gfdb0.py
gffaq.html gfinstall.py gfintrospect.py gfrecover.html
gfserve.py gfsocket.py gfstest.py gftest.py idl.py index.html
kjParseBuild.py kjParser.py kjSet.py kjbuckets0.py kjpylint.py
kwParsing.html pygram.py relalg.py remotetest.py server.html
sql.mar sqlbind.py sqlgen.py sqlgram.py sqlgtest.py sqlmod.py
sqlsem.py sqlwhere.py
Log Message:
Grand renaming:
- Renamed most files (especially python modules) to lower case.
- Moved views and interfaces into separate hierarchies within each
project, where each top-level directory under the zope package
is a separate project.
- Moved everything to src from lib/python.
lib/python will eventually go away. I need access to the cvs
repository to make this happen, however.
There are probably some bits that are broken. All tests pass
and zope runs, but I haven't tried everything. There are a number
of cleanups I'll work on tomorrow.
=== Zope3/src/zope/app/rdb/gadfly/COPYRIGHT 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/COPYRIGHT Wed Dec 25 09:13:12 2002
@@ -0,0 +1,39 @@
+
+The following copyright is modified from the python copyright.
+
+Copyright Notice
+----------------
+
+The kjParsing source is copyrighted, but you can freely use and copy it
+as long as you don't change or remove the copyright:
+
+Copyright Aaron Robert Watters, 1994
+
+ All Rights Reserved
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appears in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation.
+
+AARON ROBERT WATTERS DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL AARON ROBERT WATTERS BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+Signature
+---------
+ Aaron Robert Watters
+ Department of Computer and Information Sciences
+ New Jersey Institute of Technology
+ University Heights
+ Newark, NJ 07102
+ phone (201)596-2666
+ fax (201)596-5777
+ home phone (908)545-3367
+ email: aaron@vienna.njit.edu
=== Zope3/src/zope/app/rdb/gadfly/DLispShort.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/DLispShort.py Wed Dec 25 09:13:12 2002
@@ -0,0 +1,159 @@
+# Grammar generation
+# for lisp lists with strings, ints, vars, print, and setq
+
+# set this variable to regenerate the grammar on each load
+REGENERATEONLOAD = 1
+
+import string
+GRAMMARSTRING ="""
+ Value :: ## indicates Value is the root nonterminal for the grammar
+ @R SetqRule :: Value >> ( setq var Value )
+ @R ListRule :: Value >> ( ListTail
+ @R TailFull :: ListTail >> Value ListTail
+ @R TailEmpty :: ListTail >> )
+ @R Varrule :: Value >> var
+ @R Intrule :: Value >> int
+ @R Strrule :: Value >> str
+ @R PrintRule :: Value >> ( print Value )
+"""
+COMPILEDFILENAME = "TESTLispG.py"
+MARSHALLEDFILENAME = "TESTLispG.mar"
+LISPCOMMENTREGEX = ";.*"
+INTREGEX = "["+string.digits+"]+"
+STRREGEX = '"[^\n"]*"'
+VARREGEX = "["+string.letters+"]["+string.letters+string.digits+"]*"
+
+### declare interpretation functions and regex's for terminals
+def intInterp( str ):
+ return string.atoi(str)
+def stripQuotes( str ):
+ return str[1:len(str)-1]
+def echo(string):
+ return string
+def DeclareTerminals(Grammar):
+ Grammar.Addterm("int", INTREGEX, intInterp)
+ Grammar.Addterm("str", STRREGEX, stripQuotes)
+ Grammar.Addterm("var", VARREGEX, echo)
+
+### declare the rule reduction interpretation functions.
+def EchoValue( list, Context ):
+ return list[0]
+def VarValue( list, Context ):
+ varName = list[0]
+ if Context.has_key(varName):
+ return Context[varName]
+ else:
+ raise NameError, "no such lisp variable in context "+varName
+def NilTail( list, Context ):
+ return []
+def AddToList( list, Context ):
+ return [ list[0] ] + list[1]
+def MakeList( list, Context ):
+ return list[1]
+def DoSetq( list, Context):
+ Context[ list[2] ] = list[3]
+ return list[3]
+def DoPrint( list, Context ):
+ print list[2]
+ return list[2]
+def BindRules(Grammar):
+ Grammar.Bind( "Intrule", EchoValue )
+ Grammar.Bind( "Strrule", EchoValue )
+ Grammar.Bind( "Varrule", VarValue )
+ Grammar.Bind( "TailEmpty", NilTail )
+ Grammar.Bind( "TailFull", AddToList )
+ Grammar.Bind( "ListRule", MakeList )
+ Grammar.Bind( "SetqRule", DoSetq )
+ Grammar.Bind( "PrintRule", DoPrint )
+
+# This function generates the grammar and dumps it to a file.
+def GrammarBuild():
+ import kjParseBuild
+ LispG = kjParseBuild.NullCGrammar()
+ LispG.SetCaseSensitivity(0) # grammar is not case sensitive for keywords
+ DeclareTerminals(LispG)
+ LispG.Keywords("setq print")
+ LispG.punct("().")
+ LispG.Nonterms("Value ListTail")
+ LispG.comments([LISPCOMMENTREGEX])
+ LispG.Declarerules(GRAMMARSTRING)
+ LispG.Compile()
+
+ print "dumping as python to "+COMPILEDFILENAME
+ outfile = open(COMPILEDFILENAME, "w")
+ LispG.Reconstruct("LispG",outfile,"GRAMMAR")
+ outfile.close()
+
+ print "dumping as binary to "+MARSHALLEDFILENAME
+ outfile = open(MARSHALLEDFILENAME, "w")
+ LispG.MarshalDump(outfile)
+ outfile.close()
+
+ BindRules(LispG)
+ return LispG
+
+# this function initializes the compiled grammar from the generated file.
+def LoadLispG():
+ import TESTLispG
+ # reload to make sure we get the most recent version!
+ # (only needed when debugging the grammar).
+ reload(TESTLispG)
+ LispG = TESTLispG.GRAMMAR()
+ DeclareTerminals(LispG)
+ BindRules(LispG)
+ return LispG
+
+def unMarshalLispG():
+ import kjParser
+ infile = open(MARSHALLEDFILENAME, "r")
+ LispG = kjParser.UnMarshalGram(infile)
+ infile.close()
+ DeclareTerminals(LispG)
+ BindRules(LispG)
+ return LispG
+
+########## test the grammar generation
+if REGENERATEONLOAD:
+ print "(re)generating the LispG grammar in file TESTLispG.py"
+ Dummy = GrammarBuild()
+ print "(re)generation done."
+
+print "loading grammar as python"
+LispG = LoadLispG()
+### declare an initial context, and do some tests.
+Context = { 'x':3 }
+test1 = LispG.DoParse1( '()', Context)
+test2 = LispG.DoParse1( '(123)', Context)
+test3 = LispG.DoParse1( '(x)', Context)
+test4 = LispG.DoParse1( '" a string "', Context)
+test5 = LispG.DoParse1( '(setq y (1 2 3) )', Context )
+test6 = LispG.DoParse1( '(SeTq x ("a string" "another" 0))', Context )
+test7str = """
+ ; this is a lisp comment
+ (setq abc (("a" x)
+ ("b" (setq d 12))
+ ("c" y) ) ; another lisp comment
+ )
+"""
+test7 = LispG.DoParse1( test7str, Context)
+test8 = LispG.DoParse1( '(print (1 x d))', Context)
+
+print "unmarshalling the grammar"
+LispG2 = unMarshalLispG()
+### declare an initial context, and do some tests.
+Context = { 'x':3 }
+test1 = LispG2.DoParse1( '()', Context)
+test2 = LispG2.DoParse1( '(123)', Context)
+test3 = LispG2.DoParse1( '(x)', Context)
+test4 = LispG2.DoParse1( '" a string "', Context)
+test5 = LispG2.DoParse1( '(setq y (1 2 3) )', Context )
+test6 = LispG2.DoParse1( '(SeTq x ("a string" "another" 0))', Context )
+test7str = """
+ ; this is a lisp comment
+ (setq abc (("a" x)
+ ("b" (setq d 12))
+ ("c" y) ) ; another lisp comment
+ )
+"""
+test7 = LispG2.DoParse1( test7str, Context)
+test8 = LispG2.DoParse1( '(print (1 x d))', Context)
=== Zope3/src/zope/app/rdb/gadfly/DumbLispGen.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/DumbLispGen.py Wed Dec 25 09:13:12 2002
@@ -0,0 +1,236 @@
+
+#
+# test for kjParseBuild module automatic parser generation
+#
+# lisp lists with strings, ints, vars, and setq
+
+import string
+
+### The string representation for the grammar.
+### Since this is used only by GrammarBuild()
+### it could be put in a separate file with GrammarBuild()
+### to save space/load time after Grammar compilation.
+###
+GRAMMARSTRING ="""
+ Value :: ## indicates Value is the root nonterminal for the grammar
+ @R SetqRule :: Value >> ( setq var Value )
+ @R ListRule :: Value >> ( ListTail
+ @R TailFull :: ListTail >> Value ListTail
+ @R TailEmpty :: ListTail >> )
+ @R Varrule :: Value >> var
+ @R Intrule :: Value >> int
+ @R Strrule :: Value >> str
+"""
+
+### the name of the file in which to create the compiled
+### grammar declarations
+COMPILEDFILENAME = "TESTLispG2.py"
+
+### declare comment form(s) as regular expressions
+LISPCOMMENTREGEX = ";.*"
+
+### declare regular expression string constants for terminals
+
+#integer terminal:::::::
+INTREGEX = "["+string.digits+"]+"
+
+#string terminal::::::::
+STRREGEX = '"[^\n"]*"'
+
+#var terminal::::::::
+VARREGEX = "["+string.letters+"]["+string.letters+string.digits+"]*"
+
+### declare interpretation functions for terminals
+
+# int interpretation function: translates string to int:
+# Could use string.atoi without the extra level of indirection
+# but for demo purposes here it is.
+#
+def intInterp( str ):
+ return string.atoi(str)
+
+# interpretation function for strings strips off the surrounding quotes.
+def stripQuotes( str ):
+ if len(str)<2:
+ TypeError, "string too short?"
+ return str[1:len(str)-1]
+
+# interpretation function for vars just returns the recognized string
+def echo(string):
+ return string
+
+# This function declares the nonterminals both in the
+# "grammar generation phase" and in loading the compiled
+# grammar after generation
+#
+def DeclareTerminals(Grammar):
+ Grammar.Addterm("int", INTREGEX, intInterp)
+ Grammar.Addterm("str", STRREGEX, stripQuotes)
+ Grammar.Addterm("var", VARREGEX, echo)
+
+### declare the rule reduction interpretation functions.
+
+# EchoValue() serves for Intrule and Strrule, since
+# we just want to echo the value returned by the
+# respective terminal interpretation functions.
+#
+# Parser delivers list of form [ interpreted_value ]
+def EchoValue( list, Context ):
+ if len(list)!=1:
+ raise TypeError, "this shouldn't happen! (1)"
+ return list[0]
+
+# for Varrule interpreter must try to look up the value
+# in the Context dictionary
+#
+# Parser delivers list of form [ var_name ]
+def VarValue( list, Context ):
+ if len(list)!=1:
+ raise TypeError, "Huh? (2)"
+ varName = list[0]
+ if Context.has_key(varName):
+ return Context[varName]
+ else:
+ raise NameError, "no such lisp variable in context "+varName
+
+# for an empty tail, return the empty list
+#
+# Parser delivers list of form [")"]
+def NilTail( list, Context ):
+ if len(list) != 1 or list[0] != ")":
+ return TypeError, "Bad reduction?"
+ return []
+
+# For a full tail, add the new element to the front of the list
+#
+# Parser delivers list of form [Value, TailValue]
+def AddToList( list, Context ):
+ if len(list) !=2:
+ return TypeError, "Bad reduction?"
+ return [ list[0] ] + list[1]
+
+# For a list, simply return the list determined by the tail
+#
+# Parser delivers list of form ["(", TailValue ]
+def MakeList( list, Context ):
+ if len(list)!=2 or list[0]!="(":
+ raise TypeError, "Bad reduction? (3)"
+ return list[1]
+
+# For a setq, declare a new variable in the Context dictionary
+#
+# Parser delivers list of form # ["(", "setq", varName, Value, ")"]
+def DoSetq( list, Context):
+ if len(list) != 5\
+ or list[0] != "("\
+ or list[1] != "setq"\
+ or list[4] != ")":
+ print list
+ raise TypeError, "Bad reduction? (4)"
+ VarName = list[2]
+ if type(VarName) != type(''):
+ raise TypeError, "Bad var name? (5)"
+ Value = list[3]
+ # add or set the variable in the Context dictionary
+ Context[ VarName ] = Value
+ return Value
+
+# This function Binds the named rules of the Grammar string to their
+# interpretation functions in a Grammar.
+#
+def BindRules(Grammar):
+ Grammar.Bind( "Intrule", EchoValue )
+ Grammar.Bind( "Strrule", EchoValue )
+ Grammar.Bind( "Varrule", VarValue )
+ Grammar.Bind( "TailEmpty", NilTail )
+ Grammar.Bind( "TailFull", AddToList )
+ Grammar.Bind( "ListRule", MakeList )
+ Grammar.Bind( "SetqRule", DoSetq )
+
+# This function generates the grammar and dumps it to a file.
+# Since it will be used only once (after debugging),
+# it probably should be put in another file save memory/load-time.
+#
+# the result returned is a Grammar Object that can be used
+# for testing/debugging purposes.
+#
+# (maybe this should be made into a generic function?)
+def GrammarBuild():
+ import kjParseBuild
+
+ # initialize a Null compilable grammar to define
+ LispG = kjParseBuild.NullCGrammar()
+
+ # declare terminals for the grammar
+ DeclareTerminals(LispG)
+
+ # declare the keywords for the grammar
+ # defun is not used, included here for demo purposes only
+ LispG.Keywords("setq defun")
+
+ # Declare punctuations
+ # dot is not used here
+ LispG.punct("().")
+
+ # Declare Nonterms
+ LispG.Nonterms("Value ListTail")
+
+ # Declare comment forms
+ LispG.comments([LISPCOMMENTREGEX])
+
+ # Declare rules
+ LispG.Declarerules(GRAMMARSTRING)
+
+ # Compile the grammar
+ LispG.Compile()
+
+ # Write the grammar to a file except for
+ # the function bindings (which must be rebound)
+ outfile = open(COMPILEDFILENAME, "w")
+ LispG.Reconstruct("LispG",outfile,"GRAMMAR")
+ outfile.close()
+
+ # for debugging purposes only, bind the rules
+ # in the generated grammar
+ BindRules(LispG)
+
+ # return the generated Grammar
+ return LispG
+
+# this function initializes the compiled grammar from
+# generated file.
+def LoadLispG():
+ import TESTLispG2
+ # make sure we have most recent version (during debugging)
+ reload(TESTLispG2)
+ # evaluate the grammar function from generated file
+ LispG = TESTLispG2.GRAMMAR()
+ # bind the semantics functions
+ DeclareTerminals(LispG)
+ BindRules(LispG)
+ return LispG
+
+########## test grammar generation
+
+# do generation
+Dummy = GrammarBuild()
+
+# load the grammar from the file as LispG
+LispG = LoadLispG()
+
+# declare an initial context, and do some tests.
+Context = { "x":3 }
+test1 = LispG.DoParse1( "()", Context)
+test2 = LispG.DoParse1( "(123)", Context)
+test3 = LispG.DoParse1( "(x)", Context)
+test4 = LispG.DoParse1( '" a string "', Context)
+test5 = LispG.DoParse1( "(setq y (1 2 3) )", Context )
+test6 = LispG.DoParse1( '(setq x ("a string" "another" 0))', Context )
+test7str = """
+ ; this is a lisp comment
+ (setq abc (("a" x)
+ ("b" (setq d 12))
+ ("c" y) ) ; another lisp comment
+ )
+"""
+test7 = LispG.DoParse1( test7str, Context)
=== Zope3/src/zope/app/rdb/gadfly/SQLTESTG.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/SQLTESTG.py Wed Dec 25 09:13:12 2002
@@ -0,0 +1,78 @@
+# this was used for debugging null productions (a nearly full sql grammar
+# is available on request).
+
+#set this to automatically rebuild the grammar.
+REBUILD = 1
+MARSHALFILE = "SQLTEST.mar"
+
+SELECTRULES = """
+ ## highest level for select statement (not select for update)
+ select-statement ::
+ @R selectR :: select-statement >>
+ SELECT
+ from-clause
+ where-clause
+ group-by-clause
+ having-clause
+ ## generalized to allow null from clause eg: select 2+2
+ @R fromNull :: from-clause >>
+ @R fromFull :: from-clause >> FROM
+ @R whereNull :: where-clause >>
+ @R whereFull :: where-clause >> WHERE
+ @R groupNull :: group-by-clause >>
+ @R groupFull :: group-by-clause >> GROUP BY
+ @R havingNull :: having-clause >>
+ @R havingFull :: having-clause >> HAVING
+ @R unionNull :: union-clause >>
+ @R unionFull :: union-clause >> UNION
+"""
+
+SELECTNONTERMS = """
+ select-statement
+ all-distinct select-list table-reference-list
+ where-clause group-by-clause having-clause union-clause
+ maybe-order-by
+ search-condition column-list maybe-all order-by-clause
+ column-name from-clause
+"""
+# of these the following need resolution
+# (select-list) (table-reference-list)
+# (search-condition) order-by-clause (column-name)
+
+SELECTKEYWORDS = """
+ SELECT FROM WHERE GROUP BY HAVING UNION DISTINCT ALL AS
+"""
+
+# test generation of the grammar
+def BuildSQLG():
+ import kjParseBuild
+ SQLG = kjParseBuild.NullCGrammar()
+ SQLG.SetCaseSensitivity(0)
+ SQLG.Keywords(SELECTKEYWORDS)
+ SQLG.Nonterms(SELECTNONTERMS)
+ # no comments yet
+ SQLG.Declarerules(SELECTRULES)
+ print "building"
+ SQLG.Compile()
+ print "marshaling"
+ outfile = open( MARSHALFILE, "w")
+ SQLG.MarshalDump(outfile)
+ outfile.close()
+ return SQLG
+
+# load function
+def LoadSQLG():
+ import kjParser
+ print "unmarshalling"
+ infile = open(MARSHALFILE, "r")
+ SQLG = kjParser.UnMarshalGram(infile)
+ infile.close()
+ return SQLG
+
+#### for testing
+if REBUILD:
+ SQLG0 = BuildSQLG()
+ print " rebuilt SQLG0 as compilable grammar"
+
+SQLG = LoadSQLG()
+print " build SQLG as reloaded grammar"
=== Zope3/src/zope/app/rdb/gadfly/__init__.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/__init__.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,11 @@
+# Package handling monstrocities
+
+import os
+import sqlwhere
+
+# Stuff filename into the namespace so that you don't have to hard code
+# this information ahead of time. Should be portable across platforms.
+sqlwhere.filename = filename = __path__[0] + os.sep + 'sql.mar'
+
+# Yank the previous namespace in so that nothing breaks
+from gadfly import *
=== Zope3/src/zope/app/rdb/gadfly/gadfly.html 1.1 => 1.2 === (570/670 lines abridged)
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/gadfly.html Wed Dec 25 09:13:13 2002
@@ -0,0 +1,666 @@
+<html>
+<head>
+<title>
+Gadfly: SQL Relational Database in Python, 1.0
+</title>
+</head>
+<body bgcolor="#ffffff">
+<table>
+<tr>
+<td><img src="gadfly.JPG"></td>
+<td>
+<h1>Gadfly: SQL Relational Database in Python, 1.0</h1>
+</td></tr></table>
+<blockquote>
+Gadfly is a simple relational database system implemented in Python
+based on the SQL Structured Query Language.
+<p>
+<b>
+<a href="#install">
+The package requires installation before use, please see the section on
+installation.</a>
+<p>
+<a href="gffaq.html">In addition to this general documentation, please
+see the Gadfly frequently asked questions</a>
+</b>
+</blockquote>
+</p>
+<p>
+<ul>
+<li>
+<a href="#what">What is it?</a>
+<li>
+<a href="#Why">Why?</a>
+<li>
+<a href="#Use">Use</a>
+<li>
+<a href="#Create">Creating a new database</a>
+<li>
+<a href="#Reconnect">Reconnecting to an existing database</a>
+<li>
+<a href="#Features">Features</a>
+<li>
+<a href="#select">The SELECT statement</a>
+<li>
+<a href="#table">Table creation and "data types"</a>
+<li>
+<a href="#other">Other supported statements</a>
[-=- -=- -=- 570 lines omitted -=- -=- -=-]
+norm | 2 | lolas
+woody | 1 | lolas
+pierre | 0 | frankies
+>>>
+</pre>
+
+<h2><a name="arch">Architecture</a></h2>
+
+The SQL grammar is described in sqlgram.py, the binding of the grammar constructs
+to semantic objects is performed in sqlbind.py, the semantic objects and their
+execution strategies is defined in sqlsem.py. The semantics uses a lot of classical
+and non-classical logic (cylindric logic, to be precise) as well as
+optimization heuristics
+to define a relatively efficient and hopefully correct implementation of SQL.
+I recommend the brave have a look at sqlsem.py for a look into the 12 years of
+research into databases, logic, and programming languages that contributed bits to
+this work. The underlying logic (in a different framework) is given in
+<pre>
+A. Watters, "Interpreting a Reconstructed Relational Calculus",
+ACM SIGMOD Proceedings, 1993, Washington DC, pp. 367-376.
+</pre>
+The most basic data structures of the implementation are given in either
+kjbuckets0.py or the faster
+<a href="http://www.chordate.com/kjbuckets/">kjbucketsmodule.c</a>,
+which implement the same
+data type signatures in Python and in a C extension to Python respectively.
+<p>
+The gadfly.py module is a simple wrapper that provides a standard DBAPI interface
+to the system. The installation script gfinstall.py attempts to install the system,
+creating the grammar file sql.mar if needed (or if "forced"). The test suite
+gftest.py (which requires a writeable directory argument) attempts to provide
+a regression test and a demonstration of the system. The SQL parser also requires
+the <a href="http://www.chordate.com/kwParsing">kwParsing</a> parser
+generation package, which consists of a number of additional python modules.
+
+<h2><a name="comments">Comments</a></h2>
+
+Please find bugs and report them to <a href="mailto:arw@ifu.net">me</a>.
+<p>
+The query engine should run faster if you have the builtin
+module kjbuckets installed. Otherwise it will use a "python imitation"
+kjbuckets0.py. In one test the test suite ran two times faster using kjbuckets.
+I suspect it will have a higher payoff for larger data sets.
+<p>
+
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+</body></html>
\ No newline at end of file
=== Zope3/src/zope/app/rdb/gadfly/gadfly.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/gadfly.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,402 @@
+
+"""main entry point for gadfly sql."""
+
+import sqlgen, sqlbind
+sql = sqlgen.getSQL()
+sql = sqlbind.BindRules(sql)
+
+error = "gadfly_error"
+verbosity = 0
+
+class gadfly:
+ """as per the DBAPI spec "gadfly" is the connection object."""
+ closed = 0
+ verbose = verbosity # debug!
+
+ def __init__(self, databasename=None, directory=None,
+ forscratch=0, autocheckpoint=1, verbose=0):
+ verbose = self.verbose = self.verbose or verbose
+ # checkpoint on each commit if set
+ self.autocheckpoint = autocheckpoint
+ if verbose:
+ print "initializing gadfly instance", (\
+ databasename, directory, forscratch, verbose)
+ self.is_scratch = forscratch
+ self.databasename=databasename
+ self.directory = directory
+ self.fs = None
+ self.database = None
+ # db global transaction id
+ self.transid = 0
+ if databasename is not None:
+ self.open()
+
+ def transaction_log(self):
+ from gfdb0 import Transaction_Logger
+ if self.verbose:
+ print "new transaction log for", self.transid
+ return Transaction_Logger(self.database.log, self.transid, self.is_scratch)
+
+ # causes problems in 1.5?
+ #def __del__(self):
+ # """clear database on deletion, just in case of circularities"""
+ # # implicit checkpoint
+ # if self.verbose:
+ # print "deleting gadfly instance", self.databasename
+ # if not self.closed:
+ # self.close()
+
+ def checkpoint(self):
+ """permanently record committed updates"""
+ # note: No transactions should be active at checkpoint for this implementation!
+ # implicit abort of active transactions!
+ verbose = self.verbose
+ if verbose:
+ print "checkpointing gadfly instance", self.databasename
+ db = self.database
+ log = db.log
+ # dump committed db to fs
+ fs = self.fs
+ if fs and db and not db.is_scratch:
+ # flush the log
+ if log:
+ if verbose: print "gadfly: committing log"
+ log.commit()
+ elif verbose:
+ print "gadfly: no log to commit"
+ if verbose: print "gadfly: dumping mutated db structures"
+ fs.dump(db)
+ elif verbose:
+ print "gadfly: no checkpoint required"
+ if verbose:
+ print "gadfly: new transid, reshadowing"
+ self.transid = self.transid+1
+ self.working_db.reshadow(db, self.transaction_log())
+
+ def startup(self, databasename, directory, scratch=0, verbose=0):
+ from gfdb0 import Database0, File_Storage0
+ verbose = self.verbose
+ if verbose:
+ print "gadfly: starting up new ", databasename
+ if self.database:
+ raise error, "cannot startup, database bound"
+ self.databasename=databasename
+ self.directory = directory
+ db = self.database = Database0()
+ db.is_scratch = scratch or self.is_scratch
+ self.fs = File_Storage0(databasename, directory)
+ self.working_db = Database0(db, self.transaction_log())
+ # commit initializes database files and log structure
+ self.commit()
+ # for now: all transactions serialized
+ # working db shared among all transactions/cursors
+ self.transid = self.transid+1
+ self.working_db = Database0(db, self.transaction_log())
+
+ def restart(self):
+ """reload and rerun committed updates from log, discard uncommitted"""
+ # mainly for testing recovery.
+ if self.verbose:
+ print "gadfly: restarting database", self.databasename
+ self.database.clear()
+ self.working_db.clear()
+ self.working_db = None
+ self.database = None
+ self.open()
+
+ def open(self):
+ """(re)load existing database"""
+ if self.verbose:
+ print "gadfly: loading database", self.databasename
+ from gfdb0 import File_Storage0, Database0
+ if self.directory:
+ directory = self.directory
+ else:
+ directory = "."
+ fs = self.fs = File_Storage0(self.databasename, directory)
+ db = self.database = fs.load(sql)
+ self.transid = self.transid+1
+ self.working_db = Database0(db, self.transaction_log())
+
+ def add_remote_view(self, name, definition):
+ """add a remote view to self.
+ Must be redone on each reinitialization!
+ Must not recursively reenter the query evaluation process for
+ this database!
+ "Tables" added in this manner cannot be update via SQL.
+ """
+ self.database[name] = definition
+ self.working_db[name] = definition
+
+ def close(self):
+ """checkpoint and clear the database"""
+ if self.closed: return
+ if self.verbose:
+ print "gadfly: closing database", self.databasename
+ db = self.database
+ if not db.is_scratch:
+ self.checkpoint()
+ if db: db.clear()
+ wdb = self.working_db
+ if wdb:
+ wdb.clear()
+ self.working_db = None
+ self.closed = 1
+
+ def commit(self):
+ """commit the working database+transaction, flush log, new transid"""
+ verbose = self.verbose
+ autocheckpoint = self.autocheckpoint
+ if self.verbose:
+ print "gadfly: committing", self.transid, self.databasename
+ self.transid = self.transid+1
+ fs = self.fs
+ db = self.database
+ wdb = self.working_db
+ wdblog = wdb.log
+ if wdblog: wdblog.commit()
+ wdb.commit()
+ if fs and db and not db.is_scratch:
+ if autocheckpoint:
+ if verbose:
+ print "gadfly: autocheckpoint"
+ # skips a transid?
+ self.checkpoint()
+ else:
+ if verbose:
+ print "gadfly: no autocheckpoint"
+ wdb.reshadow(db, self.transaction_log())
+ else:
+ if verbose:
+ print "gadfly: scratch db, no logging, just reshadow"
+ wdb.reshadow(db, self.transaction_log())
+
+ def rollback(self):
+ """discard the working db, new transid, recreate working db"""
+ verbose = self.verbose
+ if verbose:
+ print "gadfly: rolling back", self.transid, self.databasename
+ if not (self.fs or self.database):
+ raise error, "unbound, cannot rollback"
+ # discard updates in working database
+ self.working_db.clear()
+ self.transid = self.transid+1
+ self.working_db.reshadow(self.database, self.transaction_log())
+ #self.open()
+
+ def cursor(self):
+ if self.verbose:
+ print "gadfly: new cursor", self.databasename
+ db = self.database
+ if db is None:
+ raise error, "not bound to database"
+ return GF_Cursor(self)
+
+ def dumplog(self):
+ log = self.database.log
+ if log:
+ log.dump()
+ else:
+ print "no log to dump"
+
+ def table_names(self):
+ return self.working_db.relations()
+
+ def DUMP_ALL(self):
+ print "DUMPING ALL CONNECTION DATA", self.databasename, self.directory
+ print
+ print "***** BASE DATA"
+ print
+ print self.database
+ print
+ print "***** WORKING DATA"
+ print
+ print self.working_db
+
+
+class GF_Cursor:
+
+ verbose = verbosity
+
+ arraysize = None
+
+ description = None
+
+ EVAL_DUMP = 0 # only for extreme debugging!
+
+ def __init__(self, gadfly_instance):
+ verbose = self.verbose = self.verbose or gadfly_instance.verbose
+ if verbose:
+ print "GF_Cursor.__init__", id(self)
+ self.connection = gadfly_instance
+ self.results = None
+ self.resultlist = None
+ self.statement = None
+ # make a shadow of the shadow db! (in case of errors)
+ from gfdb0 import Database0
+ self.shadow_db = Database0()
+ self.reshadow()
+ self.connection = gadfly_instance
+
+ def reshadow(self):
+ if self.verbose:
+ print "GF_Cursor.reshadow", id(self)
+ db = self.connection.working_db
+ shadow = self.shadow_db
+ shadow.reshadow(db, db.log)
+ if self.verbose:
+ print "rels", shadow.rels.keys()
+
+ def close(self):
+ if self.verbose:
+ print "GF_Cursor.close", id(self)
+ self.connection = None
+
+ def reset_results(self):
+ if self.verbose:
+ print "GF_Cursor.reset_results", id(self)
+ rs = self.results
+ if rs is None:
+ raise error, "must execute first"
+ if len(rs)!=1:
+ raise error, "cannot retrieve multiple results"
+ rel = rs[0]
+ rows = rel.rows()
+ atts = rel.attributes()
+ tupatts = tuple(atts)
+ resultlist = list(rows)
+ if len(tupatts)==1:
+ att = tupatts[0]
+ for i in xrange(len(resultlist)):
+ resultlist[i] = (resultlist[i][att],)
+ else:
+ for i in xrange(len(resultlist)):
+ resultlist[i] = resultlist[i].dump(tupatts)
+ self.resultlist = resultlist
+
+ def fetchone(self):
+ if self.verbose:
+ print "GF_Cursor.fetchone", id(self)
+ r = self.resultlist
+ if r is None:
+ self.reset_results()
+ r = self.resultlist
+ if len(r)<1:
+ raise error, "no more results"
+ result = r[0]
+ del r[0]
+ return result
+
+ def fetchmany(self, size=None):
+ if self.verbose:
+ print "GF_Cursor.fetchmany", id(self)
+ r = self.resultlist
+ if r is None:
+ self.reset_results()
+ r = self.resultlist
+ if size is None:
+ size = len(r)
+ result = r[:size]
+ del r[:size]
+ return result
+
+ def fetchall(self):
+ if self.verbose:
+ print "GF_Cursor.fetchall", id(self)
+ return self.fetchmany()
+
+ def execute(self, statement=None, params=None):
+ """execute operations, commit results if no error"""
+ success = 0
+ verbose = self.verbose
+ if verbose:
+ print "GF_Cursor.execute", id(self)
+ if statement is None and self.statement is None:
+ raise error, "cannot execute, statement not bound"
+ if statement!=self.statement:
+ if verbose: print "GF_cursor: new statement: parsing"
+ # only reparse on new statement.
+ self.statement=statement
+ from sqlsem import Parse_Context
+ context = Parse_Context()
+ cs = self.commands = sql.DoParse1(statement, context)
+ else:
+ if verbose: print "GF_cursor: old statment, not parsing"
+ cs = self.commands
+ # always rebind! (db may have changed)
+ if verbose: print "GF_Cursor: binding to temp db"
+ # make a new shadow of working db
+ # (should optimize?)
+ self.reshadow()
+ # get shadow of working database
+ database = self.shadow_db
+ if self.EVAL_DUMP:
+ print "***"
+ print "*** dumping connection parameters before eval"
+ print "***"
+ print "*** eval scratch db..."
+ print
+ print database
+ print
+ print "*** connection data"
+ print
+ self.connection.DUMP_ALL()
+ print "********** end of eval dump"
+ for i in xrange(len(cs)):
+ if verbose:
+ print "GFCursor binding\n", cs[i]
+ print database.rels.keys()
+ cs[i] = cs[i].relbind(database)
+ cs = self.commands
+ self.results = results = list(cs)
+ # only unshadow results on no error
+ try:
+ for i in xrange(len(cs)):
+ results[i] = cs[i].eval(params)
+ success = 1
+ finally:
+ #print "in finally", success
+ # only on no error...
+ if success:
+ # commit updates in shadow of working db (not in real db)
+ if verbose: print "GFCursor: successful eval, storing results in wdb"
+ database.log.flush()
+ # database commit does not imply transaction commit.
+ database.commit()
+ else:
+ if verbose:
+ print \
+ "GFCursor: UNSUCCESSFUL EVAL, discarding results and log entries"
+ self.statement = None
+ self.results = None
+ self.resultlist = None
+ database.log.reset()
+ # handle curs.description
+ self.description = None
+ if len(results)==1:
+ result0 = results[0]
+ try:
+ atts = result0.attributes()
+ except:
+ pass
+ else:
+ descriptions = list(atts)
+ fluff = (None,) * 6
+ for i in xrange(len(atts)):
+ descriptions[i] = (atts[i],) + fluff
+ self.description = tuple(descriptions)
+ self.resultlist = None
+
+ def setoutputsize(self, *args):
+ # not implemented
+ pass
+
+ def setinputsizes(self, *args):
+ # not implemented
+ pass
+
+ def pp(self):
+ """return pretty-print string rep of current results"""
+ from string import join
+ stuff = map(repr, self.results)
+ return join(stuff, "\n\n")
=== Zope3/src/zope/app/rdb/gadfly/gfSQL.html 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfSQL.html Wed Dec 25 09:13:13 2002
@@ -0,0 +1,331 @@
+<html>
+<head>
+<title>
+Gadfly SQL constructs
+</title>
+</head>
+<body bgcolor="#ffffff">
+
+<table>
+<tr>
+<td>
+<img src="gadfly.JPG">
+</td>
+<td>
+<h1>Gadfly SQL constructs</h1>
+</td></tr></table>
+
+<blockquote>
+This document describes SQL constructs supported by Gadfly.
+The presentation
+does not define the complete syntax -- see sqlgram.py for
+the precise syntax as BNF -- nor the complete semantics --
+see a good book on SQL for more detailed coverage of semantic
+(or use the source, Luke ;c) ).
+Also, please have a look at my
+<a href="http://mulder.rutgers.edu/~aaron/dbnotes.cgi">evolving
+database course notes</a> for more coverage of SQL.
+Examples of all supported constructs are also shown in the
+test suite source file gftest.py.
+This document is only
+a very brief guide, primarily of use to those who already
+understand something about SQL -- it is neither a tutorial
+nor a detailed discussion of syntax and semantics.
+</blockquote>
+
+<h1>The Standard, with omissions</h1>
+
+<p>
+Gadfly supports a large subset of ODBC 2.0 SQL. The reason
+ODBC 2.0 is chosen is because it provides a fairly strong set of
+constructs, but does not include some of the more obscure
+features of other SQL standards which would be extremely
+difficult and complex to implement correctly
+(and perhaps, not used very frequently (?)).
+<p>
+Supported features include views, groupings, aggregates,
+subquery expressions, quantified subquery comparisons,
+EXISTS, IN, UNION, EXCEPT, INTERSECT, searched mutations and
+Indices, among others (see below).
+
+<p>
+Some important omissions from ODBC 2.0 at this point are
+<pre>
+Nulls.
+Outer joins.
+CHECK conditions.
+Enforced data type constraints.
+Alter table (can't implement until NULLs arrive).
+Date, Time, and Interval data types
+</pre>
+It is hoped these will be implemented at some future time.
+<p>
+Less important omissions include
+<pre>
+Cursor based updates and deletes
+ (justification: if you really need them the db design
+ is flawed, and it's possible to use python instead).
+LIKE string predicate
+ (justification: use Python regexes in python code).
+Users and permissions
+ (justification: Gadfly is not intended for full multiuser
+ use at this time).
+</pre>
+These may or may not be implemented at some future time.
+
+<h1>Statements</h1>
+
+All interaction with SQL databases is mediated by
+SQL statements, or statement sequences. Statement
+sequences are statements separated by semicolons.
+SQL keywords and user defined names are not case
+sensitive (but string values are, of course).
+<p>
+SQL statements include the following.
+
+<h3>Select Statement</h3>
+The select statement derives a table from tables
+in the database. It's general form is
+<pre>
+sub_query
+ optorder_by
+</pre>
+Where sub_query is given by
+<pre>
+SELECT alldistinct select_list
+ FROM table_reference_list
+ optwhere
+ optgroup
+ opthaving
+ optunion
+</pre>
+Read the statement:
+<pre>
+SELECT [DISTINCT|ALL] expressions or *
+FROM tables
+[WHERE condition]
+[GROUP BY group-expressions]
+[HAVING aggregate-condition]
+[union-clause]
+[ORDER BY columns]
+</pre>
+as follows:
+<pre>
+1) Make all combinations of rows from the tables (FROM line)
+2) Eliminate those combinations not satisfying condition (WHERE line)
+3) (if GROUP present) form aggregate groups that match on group-expressions
+4) (if HAVING present) eliminate aggregate groups that don't satisfy
+ the aggregate-condition.
+5) compute the columns to keep (SELECT line).
+6) (if union-clause present) combine (union, except, intersect)
+ the result with the result of another select statement.
+7) if DISTINCT, throw out redundant entries.
+8) (if ORDER present) order the result by the columns (ascending
+ or descending as specified, with precedence as listed).
+</pre>
+This reading has little to do with the actual implementation,
+but the answer produced should match this intuitive reading.
+
+<h3>Create and drop table</h3>
+
+The create and drop table constructs
+initialize and destroy a table structure, respectively.
+<pre>
+CREATE TABLE user_defined_name ( colelts )
+
+DROP TABLE user_defined_name
+</pre>
+The colelts declare the names of the columns for
+the table and their data types. The data types are
+not checked or enforced in any way at this time.
+
+<h3>Table mutations (INSERT, UPDATE, DELETE)</h3>
+
+Insert, Update, and Delete statements insert rows
+into tables, modify rows in tables in place, or
+remove rows from tables respectively.
+<pre>
+INSERT INTO table_name optcolids insert_spec
+
+DELETE FROM user_defined_name optwhere
+
+UPDATE user_defined_name
+ SET assns
+ optwhere
+</pre>
+The insert statement has two variants (in this implementation)
+INSERT sub-select and INSERT VALUES.
+<pre>
+insert into r (a,b,c) select a,b,c from s
+
+insert into r (a,b,c) values (1,2,3)
+</pre>
+The first inserts
+the result of a SELECT statement into the target table
+and the other inserts explicit values (which may be dynamic
+parameters, see below).
+<P>
+Cursor based updates are not supported at the SQL level,
+eg
+<pre>
+ update r set a=1 where current of curs
+</pre>
+is not supported.
+
+<h3>Indices</h3>
+The create and drop index statements initialize and
+destroy index structures respectively.
+<pre>
+CREATE INDEX user_defined_name
+ ON user_defined_name
+ ( namelist )
+
+DROP INDEX user_defined_name
+</pre>
+Indices allow fast access to a table, based on values
+for the indexed columns in the namelist.
+<p>
+Indices can be UNIQUE, meaning that the attributes
+of the index cannot take on the same values in the table
+twice.
+<pre>
+CREATE UNIQUE INDEX user_defined_name
+ ON user_defined_name ( namelist )
+</pre>
+Unique indices can be used to enforce primary and
+secondary key constraints. After a UNIQUE index on
+a table is created inserts that attempt to insert
+repeat values for the indexed columns will be rejected.
+
+<h3>Views</h3>
+
+Create view and drop view statements initialize and
+drop views, respectively.
+<pre>
+CREATE VIEW user_defined_name optnamelist
+ AS select_statement
+
+DROP VIEW user_defined_name
+</pre>
+Views are "derived tables" which are defined
+as stored SELECT statements. They can be used
+as tables, except that they cannot be directly
+mutated.
+<p>
+It is possible to "implement your own views
+in Python". Please see remotetest.py, gfintrospect
+and the FAQ for discussion.
+
+<h1>Conditions</h1>
+
+Conditions are truth valued boolean expressions
+formed from basic conditions possibly combined using
+NOT, AND, OR (where NOT has highest precedence and
+OR has lowest precedence) and parentheses.
+<p>
+Basic conditions include simple comparisons
+<pre>
+expression = expression
+expression < expression
+expression > expression
+expression <= expression
+expression >= expression
+expression <> expression
+</pre>
+Variants of the simple comparisons are the quantified
+subquery comparisons
+<pre>
+expression = ANY ( subquery )
+expression = ALL ( subquery )
+</pre>
+(and similarly for the other comparison operators).
+The IN predicate tests membership (like =ANY)
+<pre>
+expression IN ( subquery )
+expression NOT IN ( subquery )
+</pre>
+For all the quantified comparisons and IN the
+subquery must generate a single column table.
+<p>
+Also included are the the BETWEEN and NOT BETWEEN predicates
+<pre>
+expression BETWEEN expression AND expression
+expression NOT BETWEEN expression AND expression
+</pre>
+<p>
+The most general subquery predicate is EXISTS and NOT EXISTS
+which places no restriction on the subquery:
+<pre>
+EXISTS (subquery)
+NOT EXISTS (subquery)
+</pre>
+
+<h1>Expressions</h1>
+
+Expressions occur in conditions (WHERE, HAVING, etc.),
+in UPDATE searched assignments,
+and in the select list of select statements.
+<p>
+Expressions are formed from primary expressions,
+possibly combined using the standard arithmetic operators
+and parentheses with the normal precedence.
+<p>
+Primary expressions include numeric and string literals.
+Numeric literals supported are the Python numeric literals.
+String constants are set off by apostrophies, where two
+apostrophe's in sequence represent an apostrophy in the
+string:
+<pre>
+'SQL string literals ain''t pretty'
+</pre>
+Column name expressions may be unqualified if they are
+unambiguous, or may be qualified with a table name
+or table alias
+<pre>
+bar
+frequents.bar
+f.bar
+</pre>
+The rules for scoping of column names are not covered
+here. Column names in subqueries may refer to bindings
+in the query (or queries) that contain the sub-query.
+<p>
+Subquery expressions of form
+<pre>
+( select_statement )
+</pre>
+must produce a single column and single row table.
+<p>
+Aggregate operations are only permitted in the select
+list or in the HAVING condition of SELECT statements
+(including subselects).
+<pre>
+COUNT(*)
+COUNT(expression)
+AVG(expression)
+MAX(expression)
+SUM(expression)
+MIN(expression)
+</pre>
+<em><strong>and also including the non-standard extension MEDIAN
+<pre>
+MEDIAN(expression)
+</pre>
+</strong></em>
+Aggregate operations can be applied to distinct values
+as in
+<pre>
+COUNT(DISTINCT expression)
+</pre>
+The Dynamic expression "?" is a placeholder for a value
+bound at evaluation time (from Python values). See the
+<a href="gadfly.html">
+API discussions </a>
+(Use) for more details on the use of
+dynamic parameters.
+<p>
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+</body>
+</html>
\ No newline at end of file
=== Zope3/src/zope/app/rdb/gadfly/gfclient.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfclient.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,217 @@
+
+"""client access to gadfly server. (gfserve.py)
+
+Imported as a module this module provides interfaces
+that remotely access a gadfly database server.
+
+Remote connections: gfclient
+
+ connection = gfclient.gfclient(
+ policy, # the name of the connection policy ["admin" for admin]
+ port, # the port number the server is running on
+ password,# the password of the policy
+ [machine]) # (optional) the machine where server is running
+ # (defaults to localhost)
+
+ methods for gfclient connections:
+ gfclient.checkpoint() checkpoint the server (fails silently
+ if connection is not "admin").
+ gfclient.restart() restart the server (fails silently
+ if connection is not "admin").
+ gfclient.shutdown() shutdown the server (fails silently
+ if connection is not "admin").
+ cursor = gfclient.cursor() returns a cursor on this connection
+
+ methods for cursor objects:
+
+ cursor.execute(statement, dynamic_parameters=None)
+ execute the statement with optional dynamic parameters.
+ Dynamic parameters can be a list of tuples for INSERT
+ VALUES statements, otherwise they must be a tuple
+ of values.
+ cursor.execute_prepared(name, dynamic_parameters=None)
+ execute a named statement configured for this connection
+ policy, with optional dynamic parameters. Dynamic
+ parameters permitted as for execute depending on the
+ statement the name refers to.
+ cursor.fetchall()
+ return results of the last executed statement
+ (None for non-queries, or list of tuples).
+
+See gfstest.py for example usage.
+
+SCRIPT INTERPRETATION:
+
+Server maintenance utilities
+
+COMMAND LINE:
+ python gfclient.py action port admin_password [machine]
+
+TEST EXAMPLE:
+ python gfclient.py shutdown 2222 admin
+
+ action: one of
+ shutdown: shut down the server with no checkpoint
+ restart: restart the server (re-read the database and recover)
+ checkpoint: force a database checkpoint now
+ port: the port the server is running on
+ admin_password: the administrative password for the server
+ machine: [optional] the machine the server runs on.
+"""
+
+import gfsocket
+
+def main():
+ import sys
+ try:
+ done=0
+ argv = sys.argv
+ [action, port, admin_password] = argv[1:4]
+ from string import atoi
+ port = atoi(port)
+ if len(argv)>4:
+ machine = argv[4]
+ else:
+ machine = None
+ print action, port, admin_password, machine
+ if action not in ["shutdown", "restart", "checkpoint"]:
+ print "bad action", action
+ print
+ return
+ dosimple(action, port, admin_password, machine)
+ done=1
+ finally:
+ if not done:
+ print __doc__
+
+def dosimple(action, port, pw, machine=None):
+ import socket
+ if machine is None:
+ machine = socket.gethostname()
+ conn = gfclient("admin", port, pw, machine)
+ action = getattr(conn, action)
+ print action()
+
+# copied from gfserve
+# shut down the server (admin policy only)
+# arguments = ()
+# shutdown the server with no checkpoint
+SHUTDOWN = "SHUTDOWN"
+
+# restart the server (admin only)
+# arguments = ()
+# restart the server (recover)
+# no checkpoint
+RESTART = "RESTART"
+
+# checkpoint the server (admin only)
+# arguments = ()
+# checkpoint the server
+CHECKPOINT = "CHECKPOINT"
+
+# exec prepared statement
+# arguments = (prepared_name_string, dyn=None)
+# execute the prepared statement with dynamic args.
+# autocommit.
+EXECUTE_PREPARED = "EXECUTE_PREPARED"
+
+# exec any statement (only if not disabled)
+# arguments = (statement_string, dyn=None)
+# execute the statement with dynamic args.
+# autocommit.
+EXECUTE_STATEMENT = "EXECUTE_STATEMENT"
+
+class gfclient:
+
+ closed = 0
+
+ def __init__(self, policy, port, password, machine=None):
+ import socket
+ self.policy = policy
+ self.port = port
+ self.password = password
+ if machine is None:
+ machine = socket.gethostname()
+ self.machine = machine
+
+ def open_connection(self):
+ import socket
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ #print type(sock), sock
+ sock.connect((self.machine, self.port))
+ return sock
+
+ def send_action(self, action, arguments, socket):
+ gfsocket.send_certified_action(
+ self.policy, action, arguments, self.password, socket)
+
+ def checkpoint(self):
+ return self.simple_action(CHECKPOINT)
+
+ def simple_action(self, action, args=()):
+ """only valid for admin policy: force a server checkpoint"""
+ sock = self.open_connection()
+ self.send_action(action, args, sock)
+ data = gfsocket.recv_data(sock)
+ data = gfsocket.interpret_response(data)
+ return data
+
+ def restart(self):
+ """only valid for admin policy: force a server restart"""
+ return self.simple_action(RESTART)
+
+ def shutdown(self):
+ """only valid for admin policy: shut down the server"""
+ return self.simple_action(SHUTDOWN)
+
+ def close(self):
+ self.closed = 1
+
+ def commit(self):
+ # right now all actions autocommit
+ pass
+
+ # cannot rollback, autocommit on success
+ rollback = commit
+
+ def cursor(self):
+ """return a cursor to this policy"""
+ if self.closed:
+ raise ValueError, "connection is closed"
+ return gfClientCursor(self)
+
+
+class gfClientCursor:
+
+ statement = None
+ results = None
+ description = None
+
+ def __init__(self, connection):
+ self.connection = connection
+
+ # should add fetchone fetchmany
+ def fetchall(self):
+ return self.results
+
+ def execute(self, statement=None, params=None):
+ con = self.connection
+ data = con.simple_action(EXECUTE_STATEMENT, (statement, params))
+ (self.description, self.results) = data
+
+ def execute_prepared(self, name, params=None):
+ con = self.connection
+ data = con.simple_action(EXECUTE_PREPARED, (name, params))
+ if data is None:
+ self.description = self.results = None
+ else:
+ (self.description, self.results) = data
+
+ def setoutputsizes(self, *args):
+ pass # not implemented
+
+ def setinputsizes(self):
+ pass # not implemented
+
+if __name__=="__main__":
+ main()
=== Zope3/src/zope/app/rdb/gadfly/gfdb0.py 1.1 => 1.2 === (1315/1415 lines abridged)
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfdb0.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,1412 @@
+"""storage objects"""
+
+verbosity = 0
+
+import os
+
+# use whatever kjbuckets sqlsem is using
+#from sqlsem import kjbuckets, maketuple
+
+# error on checking of data integrity
+StorageError = "StorageError"
+
+# use md5 checksum (stub if md5 unavailable?)
+def checksum(string):
+ from md5 import new
+ return new(string).digest()
+
+def recursive_dump(data, prefix="["):
+ """for debugging"""
+ from types import StringType
+ if type(data) is StringType:
+ #print prefix, data
+ return
+ p2 = prefix+"["
+ try:
+ for x in data:
+ recursive_dump(x, p2)
+ except:
+ print prefix, data
+
+def checksum_dump(data, file):
+ """checksum and dump marshallable data to file"""
+ #print "checksum_dump", file
+ #recursive_dump(data)
+ from marshal import dumps, dump
+ #print "data\n",data
+ storage = dumps(data)
+ checkpair = (checksum(storage), storage)
+ dump(checkpair, file)
+
+def checksum_undump(file):
+ """undump marshallable data from file, checksum"""
+ from marshal import load, loads
+ checkpair = load(file)
+ (check, storage) = checkpair
+ if checksum(storage)!=check:
+ raise StorageError, "data load checksum fails"
[-=- -=- -=- 1315 lines omitted -=- -=- -=-]
+ op
+ if verbose:
+ print "recovery successful: clearing log file"
+ self.clear()
+ if restart:
+ if verbose:
+ print "recreating empty log file"
+ self.startup()
+
+ def read_records(self, file):
+ """return log record as (index, (tid, op)) list"""
+ verbose = self.verbose
+ if verbose: print "reading log records to error"
+ import sys
+ records = {}
+ from sqlsem import deserialize
+ count = 0
+ while 1:
+ try:
+ data = checksum_undump(file)
+ except:
+ if verbose:
+ print "record read terminated with error", len(records)
+ print sys.exc_type, sys.exc_value
+ break
+ (transactionid, serial) = data
+ operation = deserialize(serial)
+ records[count] = (transactionid, operation)
+ if verbose:
+ print count, ": read for", transactionid
+ print operation
+ count = count+1
+ if verbose: print len(records), "records total"
+ records = records.items()
+ records.sort()
+ return records
+
+ def dump(self):
+ verbose = self.verbose
+ self.shutdown()
+ print "dumping log"
+ self.verbose = 1
+ try:
+ file = open(self.filename, "rb")
+ except:
+ print "DUMP FAILED, cannot open", self.filename
+ else:
+ self.read_records(file)
+ self.verbose = verbose
+ self.restart()
=== Zope3/src/zope/app/rdb/gadfly/gffaq.html 1.1 => 1.2 === (454/554 lines abridged)
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/gffaq.html Wed Dec 25 09:13:13 2002
@@ -0,0 +1,551 @@
+<HTML>
+<HEAD>
+ <TITLE>
+Chordate Solutions
+
+Gadfly FAQ
+
+</TITLE>
+</HEAD>
+<BODY>
+<TABLE width="700" BORDER="0" CELLSPACING="2" CELLPADDING="6">
+ <TR>
+ <TD VALIGN="TOP" rowspan="3">
+ <IMG SRC="gadfly.JPG" ALT="Chordate" WIDTH="143" HEIGHT="99"></TD>
+ <TD>
+ <B>Chordate Systems</B><BR>
+ Solutions
+ </TD>
+ <td valign="TOP" rowspan="2" nowrap>
+ <a href="#General">General information</a><br>
+ <a href="#Installation">Installation</a><br>
+ <a href="#Use">Use</a><br>
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+ </td>
+ </tr><tr>
+ <TD>
+ <em>
+
+Gadfly FAQ (Frequently asked questions)<p>
+ <a href="#General">General information</a><br>
+ <UL>
+ <LI><a href="#what">What is gadfly?</a>
+ <LI><a href="#version">What is the current version?</a>
+ <LI><a href="#where">Where can I get it?</a>
+ <LI><a href="#whysql">Why SQL and the relational model?</a>
+ <LI><a href="#whypython">Why Python?</a>
+ <LI><a href="#free">What is the License? Is it Free? Why?</a>
+ <LI><a href="#bundle">You mean I can bundle it into my own product
+ for free? Really?</a>
+ <LI><a href="#platform">Where does it run?</a>
+ <LI><a href="#req">What are the software/hardware requirements?</a>
+ <LI><a href="#opt">Is there a query optimizer?</a>
+ <LI><a href="#fast">Is it fast?</a>
+ <LI><a href="#limit">Are there data size limitations?</a>
+ <LI><a href="#recovery">What about recovery after crashes?</a>
[-=- -=- -=- 454 lines omitted -=- -=- -=-]
+<p>
+For example gfintrospect.DictKeyValueView will "wrap" a Python dictionary
+as a gadfly table and automatically reflect modifications
+made by an external Python program to
+the dictionary.
+<p>
+ <LI><a name="key">How do you define a primary key?</a>
+<p>
+New in 1.0 you can get the effect of a primary key by defining
+a unique index on the primary key columns
+<pre>
+create unique index pkey on person(firstname, lastname)
+</pre>
+Effectively will enforce a primary key constraint for (firstname, lastname)
+on the person table.
+<p>
+ <LI><a name="whynot">What about NULLs, Triggers and other missing stuff?</a>
+<p>
+The present release opted not to add missing standard or non-standard
+features that were likely to cause major modifications to large sections
+of the implementation, and therefore were likely to introduce bugs.
+<p>
+ <LI><a name="like">Where is the LIKE predicate?</a>
+<p>
+The LIKE predicate for string matching is still not supported at the
+SQL level. For what it's worth, it is easy to use Python's string matching
+(regex, re, string.search, etcetera)
+facilities on the result of a query. Also, for what it's worth, since
+the gadfly optimizer won't easily be able to optimize for string matching
+the "by hand" method would essentially be what gadfly would do anyway,
+without major modifications to the implementation.
+<p>
+ <LI><a name="crash">After a crash Gadfly won't recover! Help!</a>
+<p>
+This shouldn't happen, but did happen (at least once) for a previous
+release. If it happens again, you can explicitly delete the log files
+from the database directory in order to recover the database to a state
+which may or may not correspond to the state of the database at the second
+to last commit/checkpoint operation. It is possible, but not likely,
+that the database state will include some but not all updates from the
+last commit, but, to repeat, it shouldn't happen. Please report the
+problem if it occurs again.
+<p>
+ </UL>
+
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+</BODY>
+</HTML>
=== Zope3/src/zope/app/rdb/gadfly/gfinstall.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:44 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfinstall.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,66 @@
+#!/usr/local/bin/python
+
+"""Gadfly installation script.
+
+Build the sql grammar.
+
+usage
+ python <thismodule>
+for a simple install or
+ python <thismodule> force
+for a full rebuild (with grammar regeneration).
+
+In the current directory find or create sql.mar and sqlwhere.py
+where sql.mar has the marshalled grammar data structures
+for parsing sql and sqlwhere.py is a module that indicates
+where the grammar file is as value of sqlwhere.filename.
+"""
+
+marfile = "sql.mar"
+modfile = "sqlwhere.py"
+
+print __doc__
+
+from os import getcwd, path
+cwd = getcwd()
+
+modtemplate ="""
+'''this module indicates where the sql datastructures are marshalled
+ Auto generated on install: better not touch!
+'''
+
+filename = %s
+"""
+
+#wheremod = cwd + "/" + modfile
+#where = cwd + "/" + marfile
+wheremod = path.join(cwd, modfile)
+where = path.join(cwd, marfile)
+print
+print "now creating", wheremod
+f = open(wheremod, "w")
+f.write( modtemplate % (`where`,) )
+f.close()
+
+from sqlgen import BuildSQL, getSQL
+import sys
+argv = sys.argv
+force = 0
+#print argv
+if len(argv)>1 and argv[1]=="force":
+ force = 1
+if not force:
+ try:
+ sql = getSQL()
+ except:
+ print "exception", sys.exc_type, sys.exc_value
+ print "during load of SQL grammar structures."
+ print "Apparently the SQL grammar requires regeneration"
+ force = 1
+if force:
+ print "now generating parser structures (this might take a while)..."
+ #where = cwd + "/" + marfile
+ print "building in", where
+ sql = BuildSQL(where)
+print
+print "done."
=== Zope3/src/zope/app/rdb/gadfly/gfintrospect.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfintrospect.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,183 @@
+"""
+View based introspection and extension views
+"""
+
+import gfdb0
+
+class RemoteView(gfdb0.View):
+
+ """Virtual superclass. See text for methods and members to define."""
+
+ # Usually redefine __init__
+ def __init__(self):
+ pass
+
+ # set static (static=1) or dynamic (static=0)
+ # for static tuple seq is generated once per load
+ # for non-static tuple seq is regenerated once per query
+ # which uses the view.
+ static = 0
+
+ # define the column_names
+ column_names = ['Column1']
+
+ # define the row generator
+ def listing(self):
+ """return list of values (1 column)
+ or list of tuples of values (>1 column).
+ size of elts should match number of columns."""
+ return [0]
+
+ # possibly redefine __repr__ and irepr
+ def __repr__(self):
+ return "<Remote View %s at %s>" % (self.__class__, id(self))
+
+ irepr = __repr__
+
+ # for introspective methods possibly redefine relbind
+ def relbind(self, db, atts):
+ return self
+
+ ### don't touch the following unless you are a guru!
+ cached_rows = None
+
+ def uncache(self):
+ if self.static: return
+ self.cached_rows = None
+
+ def attributes(self):
+ from string import upper
+ return map(upper, self.column_names)
+
+ def rows(self, andseqs=0):
+ cached_rows = self.cached_rows
+ if cached_rows is None:
+ tups = list(self.listing())
+ from sqlsem import kjbuckets
+ undump = kjbuckets.kjUndump
+ attributes = tuple(self.attributes())
+ for i in xrange(len(tups)):
+ tups[i] = undump(attributes, tups[i])
+ cached_rows = self.cached_rows = tups
+ tups = cached_rows[:]
+ if andseqs:
+ return (tups, range(len(tups)))
+ else:
+ return tups
+
+class DualView(RemoteView):
+ """static one row one column view for testing.
+ (Inspired by Oracle DUAL relation)."""
+ # trivial example extension view
+
+ static = 1
+
+ column_names = ['Column1']
+
+ def listing(self):
+ return [0]
+
+class DictKeyValueView(RemoteView):
+ """Less trivial example. Dict keys/values converted to strings"""
+
+ static = 0 # regenerate in case dict changes
+
+ column_names = ["key", "value"]
+
+ mapstring = 1
+
+ def __init__(self, dict=None):
+ if dict is None: dict = {}
+ self.dict = dict
+
+ def listing(self):
+ items = self.dict.items()
+ if self.mapstring:
+ def mapper(item):
+ return tuple(map(str, item))
+ return map(mapper, items)
+ else:
+ return items
+
+class RelationsView(DictKeyValueView):
+ """list of relations and whether they are views."""
+
+ column_names = ["table_name", "is_view"]
+ mapstring = 0
+
+ def relbind(self, db, atts):
+ rels = db.rels
+ dict = {}
+ for relname in rels.keys():
+ dict[relname] = rels[relname].is_view
+ self.dict = dict
+ return self
+
+class IndicesView(DictKeyValueView):
+ """list of indices and relations they index"""
+
+ column_names = ["index_name", "table_name", "is_unique"]
+
+ mapstring = 0
+
+ def relbind(self, db, atts):
+ rels = db.rels
+ dict = {}
+ for relname in rels.keys():
+ rel = rels[relname]
+ if not rel.is_view:
+ index_list = rels[relname].index_list
+ for index in index_list:
+ dict[index.name] = (relname, index.unique)
+ self.dict = dict
+ return self
+
+ def listing(self):
+ L = []
+ dict = self.dict
+ keys = dict.keys()
+ for k in keys:
+ L.append( (k,) + dict[k] )
+ return L
+
+class DataDefsView(DictKeyValueView):
+ """Data defs (of non-special views) and definition dumps."""
+
+ column_names = ["name", "defn"]
+
+ mapstring = 1
+
+ def relbind(self, db, atts):
+ self.dict = db.datadefs
+ return self
+
+class ColumnsView(RemoteView):
+ """table_names and columns therein."""
+
+ column_names = ["table_name", "column_name"]
+
+ def relbind(self, db, atts):
+ rels = db.rels
+ pairs = []
+ for relname in rels.keys():
+ for att in rels[relname].attributes():
+ pairs.append( (relname, att) )
+ self.pairs = pairs
+ return self
+
+ def listing(self):
+ return self.pairs
+
+class IndexAttsView(ColumnsView):
+ """indices and attributes."""
+
+ column_names = ["index_name", "column_name"]
+
+ def relbind(self, db, atts):
+ indices = db.indices
+ pairs = []
+ for iname in indices.keys():
+ for att in indices[iname].attributes():
+ pairs.append( (iname, att) )
+ self.pairs = pairs
+ return self
=== Zope3/src/zope/app/rdb/gadfly/gfrecover.html 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfrecover.html Wed Dec 25 09:13:13 2002
@@ -0,0 +1,150 @@
+<HTML>
+<HEAD>
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=windows-1252">
+<META NAME="Generator" CONTENT="Microsoft Word 97">
+<TITLE>Gadfly: recovery</TITLE>
+<META NAME="Template" CONTENT="C:\PROGRAM FILES\MICROSOFT OFFICE\OFFICE\html.dot">
+</HEAD>
+<BODY BGCOLOR="#ffffff">
+
+<table><tr><td>
+<img src="gadfly.JPG">
+</td><td>
+<H1>Gadfly Recovery</H1>
+</td></tr></table>
+
+<P>In the event of a software glitch or crash Gadfly may terminate without having stored committed updates.
+A recovery strategy attempts to make sure
+that the unapplied commited updates are applied when the database restarts.
+It is always assumed that there is only one primary (server) process controlling the database (possibly with
+multiple clients). </P>
+
+<P>Gadfly uses a simple LOG with deferred updates recovery mechanism. Recovery should be possible in the
+presence of non-disk failures (server crash, system crash). Recovery after a disk crash is not available
+for Gadfly as yet, sorry. </P>
+
+<P>Due to portability problems Gadfly does not prevent multiple processes from "controlling" the database at
+once. For read only access multiple instances are not a problem, but for access with modification, the processes
+may collide and corrupt the database. For a read-write database, make sure only one (server) process controls
+the database at any given time. </P>
+
+<P>The only concurrency control mechanism that provides serializability for Gadfly as yet is the trivial one --
+the server serves all clients serially. This will likely change for some variant of the system at some point. </P>
+
+<P>This section explains the basic recovery mechanism. </P>
+
+<H1>Normal operation</H1>
+
+<H3>Precommit</H3>
+<P>During normal operations any active tables are in memory in the process.
+Uncommitted updates for a transaction are kept in "shadow tables" until the transaction commits using
+<pre>
+ connection.commit()
+</pre>
+The shadow tables remember the mutations that have been applied to them. The permanent table copies
+are only modified after commit time. A commit commits all updates for all cursors for the connection.
+Unless the autocommit feature is disabled (see below) a
+commit normally always triggers a checkpoint too.</P>
+
+A rollback
+<pre>
+ connection.rollback()
+</pre>
+explicitly discards all uncommitted updates and restores the connection to the previously
+committed state.</p>
+
+<P>There is a 3rd level of shadowing for statement sequences executed by a cursor.
+In particular the design attempts to make sure that if
+<pre>
+ cursor.execute(statement)
+</pre>
+fails with an error, then the shadow database will contain no updates from
+the partially executed statement (which may be a sequence of statements)
+but will reflect other completed updates that may have not been committed.
+
+<H3>Commit</H3>
+
+<P>At commit, operations applied to shadow tables are written
+out in order of application to a log file before being permanently
+applied to the active database. Finally a commit record is written to
+the log and the log is flushed. At this point the transaction is considered
+committed and recoverable, and a new transaction begins.
+Finally the values of the shadow tables replace
+the values of the permanent tables in the active database,
+(but not in the database disk files until checkpoint, if autocheckpoint
+is disabled). </P>
+
+<H3>Checkpoint</H3>
+<P>A checkpoint operation brings the persistent copies of the tables on
+disk in sync with the in-memory copies in the active database. Checkpoints
+occur at server shut down or periodically during server operation.
+The checkpoint operation runs in isolation (with no database access
+allowed during checkpoint). </P>
+
+<p><em>Note: database connections normally run a checkpoint
+after every commit, unless you set
+<pre>
+ connection.autocheckpoint = 0
+</pre>
+which asks that checkpoints be done explicitly by the program using
+<pre>
+ connection.commit() # if appropriate
+ connection.checkpoint()
+</pre>
+Explicit checkpoints should make the database perform better,
+since the disk files are written less frequently, but
+in order to prevent unneeded (possibly time consuming)
+recovery operations after a database
+is shutdown and restarted it is important to always execute an explicit
+checkpoint at server shutdown, and periodically during long server
+runs.</em></p>
+
+<p><strong>Note that if any outstanding operations are uncommitted
+at the time of a checkpoint (when autocheckpoint is disabled) the
+updates will be lost (ie, it is equivalent to a rollback).
+</strong></p>
+
+<P>At checkpoint the old persistent value of each table that has been updated since
+the last checkpoint is copied to a back up file, and the currently active value is
+written to the permanent table file. Finally if the data definitions have changed
+the old definitions are stored to a backup file and the new definitions are written
+to the permanent data definition file. To signal successful checkpoint the
+log file is then deleted.</P>
+<P>
+ At this point (after log deletion) the database is considered
+quiescent (no recovery required). Finally all back up table files are deleted.
+[Note, it might be good to keep old logs around... Comments?] </P>
+
+<P>Each table file representation is annotated with a checksum,
+so the recovery system can check that the file was stored correctly. </P>
+
+<H1>Recovery</H1>
+<P>When a database restarts it automatically determines whether
+the last active instance shut down normally and whether recovery
+is required. Gadfly discovers the need for recovery by detecting
+a non-empty current log file. </P>
+
+<P>To recover the system Gadfly first scans the log file to determine committed transactions.
+Then Gadfly rescans the log file applying the operations of committed
+transactions to the in memory table values in the order recorded.
+When reading in table values for the purpose of recovery Gadfly looks
+for a backup file for the table first. If the backup is not corrupt,
+its value is used, otherwise the permanent table file is used. </P>
+<P>After recovery Gadfly runs a normal checkpoint before resuming
+normal operation. </P>
+<p>
+<strong>
+Please note: Although I have attempted to provide a robust
+implementation
+for this software I do not guarantee its correctness. I hope
+it will work well for you but I do not assume any legal
+responsibility for problems anyone may have during use
+of these programs.
+</strong>
+
+<p>
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+</BODY>
+</HTML>
=== Zope3/src/zope/app/rdb/gadfly/gfserve.py 1.1 => 1.2 === (432/532 lines abridged)
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfserve.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,529 @@
+"""gadfly server mode
+
+ script usage
+
+ python gfserve.py port database directory password [startup]
+
+ test example
+
+ python gfserve.py 2222 test dbtest admin gfstest
+
+ port is the port to listen to
+ database is the database to start up. (must exist!)
+ directory is the directory the database is in.
+ password is the administrative access password.
+
+ startup if present should be the name of a module to use
+ for startup. The Startup module must contain a function
+
+ Dict = startup(admin_policy, connection, Server_instance)
+
+ which performs any startup actions on the database needed
+ and returns either None or a Dictionary of
+
+ name > policy objects
+
+ where the policy objects describe policies beyond the
+ admin policy. The startup function may also
+ modify the admin_policy (disabling queries for example).
+
+ The arguments passed to startup are:
+ admin_policy: the administrative policy
+ eg you could turn queries off for admin, using admin
+ only for server maintenance, or you could add prepared
+ queries to the admin_policy.
+ connection: the database connection
+ eg you could perform some inserts before server start
+ also needed to make policies.
+ Server_instance
+ Included for additional customization.
+
+ Create policies using
+ P = gfserve.Policy(name, password, connection, queries=0)
+ -- for a "secure" policy with only prepared queries allowed,
+ or
+ P = gfserve.Policy(name, password, connection, queries=1)
+ -- for a policy with full access arbitrary statement
+ execution.
[-=- -=- -=- 432 lines omitted -=- -=- -=-]
+ print_tb(tb)
+ self.reply_exception(ServerError,
+ "unexpected exception: "+exceptiondata, socket)
+ raise ServerError, exceptiondata
+ else:
+ raise ServerError, "unknown action: "+`action`
+
+ def certify(self, datastring, certificate, password):
+ # hook for subclassing
+ return certify(datastring, certificate, password)
+
+ def policy_SHUTDOWN(self, socket):
+ self.reply_success("attempting server shutdown", socket)
+ raise SHUTDOWN, "please shut down the server"
+
+ def policy_RESTART(self, socket):
+ self.reply_success("attempting server restart", socket)
+ raise RESTART, "please restart the server"
+
+ def policy_CHECKPOINT(self, socket):
+ self.reply_success("attempting server checkpoint", socket)
+ raise CHECKPOINT, "please checkpoint the server"
+
+ def policy_EXECUTE_PREPARED(self, name, dyn, socket):
+ try:
+ result = self.execute_named(name, dyn)
+ self.reply_success(result, socket)
+ except PreparedNameError, detail:
+ self.reply_exception(PreparedNameError,
+ "no such prepared statement: "+name,
+ socket)
+
+ def policy_EXECUTE_STATEMENT(self, stat, dyn, socket):
+ if not self.general_queries:
+ self.reply_exception(ServerError,
+ "general statements disallowed on this policy",
+ socket)
+ raise ServerError, "illegal statement attempt for: "+self.name
+ result = self.execute_any_statement(stat, dyn)
+ self.reply_success(result, socket)
+
+ def reply_exception(self, exc, info, socket):
+ # hook for subclassing
+ reply_exception(exc, info, socket)
+
+ def reply_success(self, data, socket):
+ # hook for subclassing
+ reply_success(data, socket)
+
+if __name__=="__main__": main()
=== Zope3/src/zope/app/rdb/gadfly/gfsocket.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfsocket.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,223 @@
+
+
+"""socket interactions for gadfly client and server"""
+
+from select import select
+
+# responses
+
+SUCCESS = "SUCCESS"
+EXCEPTION = "EXCEPTION"
+
+def reply_exception(exception, info, socket):
+ """send an exception back to the client"""
+ # any error is invisible to client
+ from gfserve import ServerError
+ import sys
+ try:
+ reply( (EXCEPTION, (exception, info)), socket)
+ except:
+ #info = "%s %s" % (sys.exc_type, sys.exc_value)
+ socket.close()
+ #raise ServerError, "reply_exception failed: "+`info`
+
+def reply_success(data, socket):
+ """report success with data back to client"""
+ reply( (SUCCESS, data), socket)
+
+def reply(data, socket):
+ from marshal import dumps
+ marshaldata = dumps(data)
+ send_packet(socket, marshaldata)
+ socket.close()
+
+def send_packet(socket, data):
+ """blast out a length marked packet"""
+ send_len(data, socket)
+ socket.send(data)
+
+def send_len(data, socket):
+ """send length of data as cr terminated int rep"""
+ info = `len(data)`+"\n"
+ socket.send(info)
+
+def send_certified_action(actor_name, action, arguments, password, socket):
+ from marshal import dumps
+ marshaldata = dumps( (action, arguments) )
+ cert = certificate(marshaldata, password)
+ #print actor_name, cert, marshaldata
+ marshaldata = dumps( (actor_name, cert, marshaldata) )
+ send_packet(socket, marshaldata)
+
+def unpack_certified_data(data):
+ from marshal import loads
+ # sanity check
+ unpack = (actor_name, certificate, marshaldata) = loads(data)
+ return unpack
+
+def recv_data(socket, timeout=10):
+ """receive data or time out"""
+ from time import time
+ endtime = time() + timeout
+ reader = Packet_Reader(socket)
+ done = 0
+ while not done:
+ timeout = endtime - time()
+ if timeout<0:
+ raise IOError, "socket time out (1)"
+ (readable, dummy, error) = select([socket], [], [socket], timeout)
+ if error:
+ raise IOError, "socket in error state"
+ if not readable:
+ raise IOError, "socket time out (2)"
+ reader.poll()
+ done = (reader.mode==READY)
+ return reader.data
+
+def interpret_response(data):
+ """interpret response data, raise exception if needed"""
+ from marshal import loads
+ (indicator, data) = loads(data)
+ if indicator==SUCCESS:
+ return data
+ elif indicator==EXCEPTION:
+ # ???
+ raise EXCEPTION, data
+ else:
+ raise ValueError, "unknown indicator: "+`indicator`
+
+# packet reader modes
+LEN = "LEN"
+DATA = "DATA"
+READY = "READY"
+ERROR = "ERROR"
+
+BLOCK_SIZE = 4028
+
+LEN_LIMIT = BLOCK_SIZE * 10
+
+class Packet_Reader:
+ """nonblocking pseudo-packet reader."""
+
+ # packets come in as decimal_len\ndata
+ # (note: cr! not crlf)
+
+ # kick too large requests if set
+ limit_len = LEN_LIMIT
+
+ def __init__(self, socket):
+ self.socket = socket
+ self.length = None
+ self.length_remaining = None
+ self.len_list = []
+ self.data_list = []
+ self.received = ""
+ self.data = None
+ self.mode = LEN
+
+ def __len__(self):
+ if self.mode is LEN:
+ raise ValueError, "still reading length"
+ return self.length
+
+ def get_data(self):
+ if self.mode is not READY:
+ raise ValueError, "still reading"
+ return self.data
+
+ def poll(self):
+ mode = self.mode
+ if mode is READY:
+ raise ValueError, "data is ready"
+ if mode is ERROR:
+ raise ValueError, "socket error previously detected"
+ socket = self.socket
+ (readable, dummy, error) = select([socket], [], [socket], 0)
+ if error:
+ self.socket.close()
+ self.mode = ERROR
+ raise ValueError, "socket is in error state"
+ if readable:
+ if mode is LEN:
+ self.read_len()
+ # note: do not fall thru automatically
+ elif mode is DATA:
+ self.read_data()
+
+ def read_len(self):
+ """assume socket is readable now, read length"""
+ socket = self.socket
+ received = self.received
+ len_list = self.len_list
+ if not received:
+ # 10 bytes at a time until len is read.
+ received = socket.recv(10)
+ while received:
+ # consume, test one char
+ input = received[0]
+ received = received[1:]
+ if input == "\n":
+ # done reading length
+ from string import join, atoi
+ try:
+ length = self.length = atoi(join(len_list, ""))
+ except:
+ self.mode = ERROR
+ socket.close()
+ raise ValueError, "bad len string? "+`len_list`
+ self.received = received
+ self.length_remaining = length
+ self.mode = DATA
+ limit_len = self.limit_len
+ if limit_len and length>limit_len:
+ raise ValueError, "Length too big: "+`(length, limit_len)`
+ return
+ if len(len_list)>10:
+ self.mode = ERROR
+ socket.close()
+ raise ValueError, "len_list too long: "+`len_list`
+ len_list.append(input)
+ if not received:
+ (readable, dummy, error) = select(\
+ [socket], [], [socket], 0)
+ if error:
+ self.mode = ERROR
+ socket.close()
+ raise ValueError, "socket in error state"
+ if readable:
+ received = socket.recv(10)
+ # remember extra data received.
+ self.received = received
+
+ def read_data(self):
+ # assume socket is readable
+ socket = self.socket
+ received = self.received
+ length_remaining = self.length_remaining
+ data_list = self.data_list
+ if received:
+ data_list.append(received)
+ self.received = ""
+ length_remaining = length_remaining - len(received)
+ recv_len = max(length_remaining, BLOCK_SIZE)
+ received = socket.recv(recv_len)
+ if received:
+ data_list.append(received)
+ length_remaining = length_remaining - len(received)
+ if length_remaining<1:
+ self.mode = READY
+ from string import join
+ self.data = join(data_list, "")
+ self.length_remaining = length_remaining
+
+def certificate(String, password):
+ """generate a certificate for a string, using a password"""
+ from md5 import new
+ if not String:
+ raise ValueError, "cannot generate certificate for empty string"
+ taggedstring = password + String
+ return new(taggedstring).digest()
+
+def certify(String, cert, password):
+ """check a certificate for a string"""
+ return certificate(String, password) == cert
=== Zope3/src/zope/app/rdb/gadfly/gfstest.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/rdb/gadfly/gfstest.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,225 @@
+
+"""test script for gadfly client and server
+
+Usage: This script interacts with the test database generated
+ by gftest.py. To start the server from the directory containing
+ the dbtest directory use:
+
+ python gfstest.py start
+
+ THIS WILL ONLY WORK IF YOU CREATED THE test DATABASE IN
+ DIRECTORY dbtest FIRST USING
+
+ python gftest.py dbtest
+
+ UNLESS YOU RUN THE SERVER IN THE BACKGROUND THE SERVER WILL
+ HOG THE WINDOW YOU STARTED IT IN AND YOU WILL HAVE TO USE ANOTHER
+ WINDOW UNTIL THE SERVER IS SHUT DOWN (SEE BELOW).
+
+ Then from *anywhere* (on the same machine) access the database
+ using
+ python gfstest.py restart
+ - restart the server (reread the database)
+ python gfstest.py checkpoint
+ - force checkpoint the server
+ python gfstest.py queries
+ - run some example queries and updates
+ python gfstest.py policy_test
+ - test the policies test and test1 created by the startup
+ function in this module.
+ python gfstest.py bogusshutdown
+ - attempt to shut down the server with a bogus password
+ [should generate an exception]
+
+...and finally
+ python gfstest.py shutdown
+ - shut down the server for real.
+
+As mentioned the startup function of this module illustrates
+how to create a "startup" function for a server and initialize policy
+objects with named, prepared queries.
+
+"""
+
+PORT = 2222
+DB = "test"
+DBDIR = "dbtest"
+PW = "admin"
+STARTUP = "gfstest"
+
+import sys, socket
+
+
+def main():
+ argv = sys.argv
+ command = argv[1]
+ if command=="start":
+ print "attempting to start the server"
+ from gfserve import Server
+ print "making a server on", PORT, DB, DBDIR, PW, STARTUP
+ S = Server(PORT, DB, DBDIR, PW, STARTUP)
+ print "initializing the server"
+ S.init()
+ print "starting the server", S.connection
+ S.start()
+ elif command=="shutdown":
+ dosimple("shutdown", PW)
+ elif command=="bogusshutdown":
+ print "BOGUS shutdown attempt"
+ dosimple("shutdown", "bad password")
+ elif command=="restart":
+ dosimple("restart", PW)
+ elif command=="checkpoint":
+ dosimple("checkpoint", PW)
+ elif command=="queries":
+ doqueries()
+ elif command=="policy_test":
+ policy_test()
+ else:
+ print "unknown command", command
+ print __doc__
+
+def policy_test():
+ """test the test1 and test policies"""
+ print "testing non-admin policies test and test1"
+ from gfclient import gfclient
+ import sys
+ machine = socket.gethostname()
+ conn = gfclient("test", PORT, "test", machine)
+ cursor = conn.cursor()
+ print "testing test policy: nan values before:"
+ cursor.execute_prepared("getnan")
+ for x in cursor.fetchall():
+ print x
+ print "updating nan"
+ cursor.execute_prepared("updatenan", ("pabst", 4))
+ print "nan after"
+ cursor.execute_prepared("getnan")
+ for x in cursor.fetchall():
+ print x
+ print "updating nan again"
+ cursor.execute_prepared("updatenan", ("rollingrock", 1))
+ print "trying an illegal update"
+ try:
+ cursor.execute("delete from frequents")
+ except:
+ print "exception", sys.exc_type, sys.exc_value
+ print "as expected"
+ else:
+ raise "DAMN!", "illegal query apparently completed!!!"
+ print; print "testing policy test1"; print
+ conn = gfclient("test1", PORT, "test1", machine)
+ cursor = conn.cursor()
+ print "getting norm"
+ cursor.execute_prepared("qlike", ("norm",))
+ print cursor.description
+ for x in cursor.fetchall():
+ print x
+ print "trying an illegal query again"
+ try:
+ cursor.execute("create table test(name varchar)")
+ except:
+ print "exception", sys.exc_type, sys.exc_value
+ print "as expected"
+ else:
+ raise "Damn!(2)", "illegal query apparently completed"
+
+def startup(admin_policy, connection, Server_instance):
+ """example startup script.
+
+ add a policies test and test1 passwords same
+ test1 is allowed to query the frequents table by name
+ test is allowed to update likes where drinker='nan'
+ also add prepared query dumpwork to admin_policy.
+ """
+ from gfserve import Policy
+ admin_policy["dumpwork"] = "select * from work"
+ test1 = Policy("test1", "test1", connection, queries=0)
+ test = Policy("test", "test", connection, queries=0)
+ test1["qlike"] = "select * from likes where drinker=?"
+ test["updatenan"] = """
+ update likes
+ set beer=?, perday=?
+ where drinker='nan'
+ """
+ test["getnan"] = """
+ select * from likes where drinker='nan'
+ """
+ return {"test": test, "test1": test1}
+
+def doqueries():
+ print "executing queries and updates"
+ from gfclient import gfclient
+ import sys
+ machine = socket.gethostname()
+ conn = gfclient("admin", PORT, PW, machine)
+ cursor = conn.cursor()
+ for q in admin_queries:
+ print;print;print q;print
+ try:
+ cursor.execute(q)
+ except:
+ print "exception in execute"
+ print sys.exc_type
+ v = sys.exc_value
+ from types import TupleType, ListType
+ if type(v) in (TupleType, ListType):
+ for x in v: print x
+ else:
+ print v
+ else:
+ print "executed"
+ #print q
+ print "description"
+ print cursor.description
+ print "results"
+ try:
+ r = cursor.fetchall()
+ if r is None:
+ print "no results"
+ else:
+ for x in r:
+ print x
+ except:
+ print "exception in results"
+ print sys.exc_type, sys.exc_value
+ print dir(cursor)
+ # try dumpwork
+ print; print; print "dumpwork"; print
+ cursor.execute_prepared("dumpwork")
+ for x in cursor.fetchall():
+ print x
+ # try dynamic parameters
+ stat = """
+ select distinct drinker
+ from likes l, serves s
+ where l.beer = s.beer and s.bar=?
+ """
+ print; print stat; print "dynamic query ?=cheers"
+ cursor.execute(stat, ("cheers",))
+ for x in cursor.fetchall():
+ print x
+
+admin_queries = [
+"""select count(*) from work""",
+"""select * from frequents""",
+"""select count(*) from frequents""",
+"""select count(drinker) from frequents""",
+"""insert into frequents(drinker, bar, perweek)
+ values ('sally', 'cheers', 2)""",
+"""select * from frequents""",
+"""select syntax error from work""",
+"""select drinker, count(bar) from frequents
+ group by drinker""",
+]
+
+def dosimple(command, pw):
+ print "attempting remote %s (%s) for server on local machine" % (command, pw)
+ from gfclient import gfclient
+ machine = socket.gethostname()
+ conn = gfclient("admin", PORT, pw, machine)
+ action = getattr(conn, command)
+ print action()
+
+if __name__=="__main__":
+ main()
=== Zope3/src/zope/app/rdb/gadfly/gftest.py 1.1 => 1.2 === (626/726 lines abridged)
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/rdb/gadfly/gftest.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,723 @@
+"""test script for gadfly
+
+usage gftest.py <directory>
+
+run in current directory creates a database in files
+ test.dfs LIKES.grl SERVES.grl FREQUENTS.grl
+"""
+
+def test(directory):
+ print "testing"
+ from gadfly import gadfly
+ connect = gadfly()
+ connect.startup("test", directory)
+ curs = connect.cursor()
+ print
+ print "TABLE CREATES"
+ for x in table_creates:
+ print x
+ curs.execute(x)
+ curs.execute("create table empty (nothing varchar)")
+ C = """
+ CREATE TABLE work (
+ name VARCHAR,
+ hours INTEGER,
+ rate FLOAT)
+ """
+ print C
+ curs.execute(C)
+ print
+ C = """
+ CREATE TABLE accesses (
+ page VARCHAR,
+ hits INTEGER,
+ month INTEGER)
+ """
+ print C
+ curs.execute(C)
+ print
+ print "INSERTS"
+ C = """
+ INSERT INTO work(name, hours, rate) VALUES (?, ?, ?)
+ """
+ D = [
+ ("sam", 30, 40.2),
+ ("norm", 45, 10.2),
+ ("woody", 80, 5.4),
+ ("diane", 3, 4.4),
[-=- -=- -=- 626 lines omitted -=- -=- -=-]
+ cursor.execute(s)
+ postresults.append(cursor.fetchall())
+ print cursor.pp()
+ except:
+ d = sys.exc_type
+ print "*** exception", d
+ postresults.append(d)
+ if preresults==postresults:
+ print "*** same results as before uncommitted updates"
+ else:
+ print "*** differing results from before uncommitted updates"
+ if dummy==1:
+ print "*** RESTART: DUMPLOG"
+ connect.dumplog()
+ print "*** RESTARTING (RECOVER FROM LOG, DISCARD UNCOMMITTED)"
+ connect.restart()
+
+def retest(directory):
+ print "*" * 30
+ print "*** reconnect test"
+ from gadfly import gadfly
+ connect = gadfly("test", directory)
+ cursor = connect.cursor()
+ for s in updates:
+ print; print
+ print s
+ if s in trace_updates:
+ cursor.EVAL_DUMP = 1
+ cursor.execute(s)
+ cursor.EVAL_DUMP = 0
+ print cursor.pp()
+ #print; print "CONNECTION DATA BEFORE COMMIT"
+ #connect.DUMP_ALL()
+ connect.commit()
+ #print; print "CONNECTION DATA AFTER COMMIT"
+ #connect.DUMP_ALL()
+ connect.close()
+ return connect
+
+if __name__=="__main__":
+ import sys
+ argv = sys.argv
+ if len(argv)<2:
+ print "USAGE: python <thismodule> <db_directory>"
+ print " please provide a directory for test database!"
+ else:
+ directory = argv[1]
+ test(directory)
+ rollbacktest(directory)
+ retest(directory)
=== Zope3/src/zope/app/rdb/gadfly/idl.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:45 2002
+++ Zope3/src/zope/app/rdb/gadfly/idl.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,447 @@
+
+# idl grammar
+#
+# Note, this grammar requires a special hack at the lexical
+# level in order to parse the fragment
+#
+# ...
+# case abc::def: jjj::www: whatever...
+#
+# (Yuck!)
+# Some would argue this is a language design flaw, but whatever...
+# It causes a shift/reduce problem without special handling for ::
+# below coloncolon is a 'fake' keyword that parses as two colons.
+
+idlgramstring = """
+
+specification ::
+
+## 1
+@R r1a :: specification >> definition speclist
+@R r1b :: speclist >> specification
+@R r1c :: speclist >>
+
+## 2 punct ;
+@R r2a :: definition >> type_dcl ;
+@R r2b :: definition >> const_dcl ;
+@R r2c :: definition >> except_dcl ;
+@R r2d :: definition >> interface_nt ;
+@R r2e :: definition >> module_nt ;
+
+## 3 identifier=term, module=kw puncts {}
+@R r3 :: module_nt >> module identifier { specification }
+
+## 4
+@R r4a :: interface_nt >> interface_dcl
+@R r4b :: interface_nt >> forward_dcl
+
+## 5
+@R r5 :: interface_dcl >> interface_header { interface_body }
+
+## 6 interface=kw
+@R r6 :: forward_dcl >> interface identifier
+
+## 7 puncts []
+@R r7 :: interface_header >> interface identifier [ inheritance_spec ]
+
+## 8
+@R r8a :: interface_body >>
+@R r8b :: interface_body >> export interface_body
+
+## 9
+@R r9a :: export >> type_dcl
+@R r9b :: export >> const_dcl
+@R r9c :: export >> except_dcl
+@R r9d :: export >> attr_dcl
+@R r9e :: export >> op_dcl
+
+## 10 punct ,:
+@R r10a :: inheritance_spec >> : scoped_name_list
+@R r10b :: scoped_name_list >> scoped_name
+@R r10c :: scoped_name_list >> scoped_name_list , scoped_name
+
+## 11
+@R r11a :: scoped_name >> identifier
+@R r11b :: scoped_name >> colon_colon identifier
+@R r11d :: scoped_name >> scoped_name coloncolon identifier
+
+## 12 const=kw punct =
+@R r12 :: const_dcl >> const const_type identifier = const_expr
+
+## 13
+@R r13a :: const_type >> integer_type
+@R r13b :: const_type >> char_type
+@R r13c :: const_type >> boolean_type
+@R r13d :: const_type >> floating_type
+@R r13e :: const_type >> string_type
+@R r13f :: const_type >> scoped_name
+
+## 14
+@R r14 :: const_expr >> or_expr
+
+##15 punct |
+@R r15a :: or_expr >> xor_expr
+@R r15b :: or_expr >> or_expr | xor_expr
+
+##16 punct ^
+@R r16a :: xor_expr >> and_expr
+@R r16b :: xor_expr >> xor_expr ^ and_expr
+
+##17 punct &
+@R r17a :: and_expr >> shift_expr
+@R r17b :: and_expr >> and_expr & shift_expr
+
+##18 punct > <
+@R r18a :: shift_expr >> add_expr
+@R r18b :: shift_expr >> shift_expr > > add_expr
+@R r18c :: shift_expr >> shift_expr < < add_expr
+
+##19 punct +-
+@R r19a :: add_expr >> mult_expr
+@R r19b :: add_expr >> add_expr + mult_expr
+@R r19c :: add_expr >> add_expr - mult_expr
+
+##20 punct */%
+@R r20a :: mult_expr >> unary_expr
+@R r20b :: mult_expr >> mult_expr * unary_expr
+@R r20c :: mult_expr >> mult_expr / unary_expr
+@R r20d :: mult_expr >> mult_expr % unary_expr
+
+##21
+@R r21a :: unary_expr >> unary_operator primary_expr
+@R r21b :: unary_expr >> primary_expr
+
+##22
+@R r22a :: unary_operator >> -
+@R r22b :: unary_operator >> +
+@R r22c :: unary_operator >> ~
+
+##23 punct ()
+@R r23a :: primary_expr >> scoped_name
+@R r23b :: primary_expr >> literal
+@R r23c :: primary_expr >> ( const_expr )
+
+##24 terms = *_literal (?) except boolean
+@R r24a :: literal >> integer_literal
+@R r24b :: literal >> string_literal
+@R r24c :: literal >> character_literal
+@R r24d :: literal >> floating_pt_literal
+@R r24e :: literal >> boolean_literal
+
+##25 kw TRUE FALSE
+@R r25a :: boolean_literal >> TRUE
+@R r25b :: boolean_literal >> FALSE
+
+##26
+@R r26 :: positive_int_literal >> const_expr
+
+##27 kw typedef
+@R r27a :: type_dcl >> typedef type_declarator
+@R r27b :: type_dcl >> struct_type
+@R r27c :: type_dcl >> union_type
+@R r27d :: type_dcl >> enum_type
+
+##28
+@R r28 :: type_declarator >> type_spec declarators
+
+##29
+@R r29a :: type_spec >> simple_type_spec
+@R r29b :: type_spec >> constr_type_spec
+
+##30
+@R r30a :: simple_type_spec >> base_type_spec
+@R r30b :: simple_type_spec >> template_type_spec
+@R r30c :: simple_type_spec >> scoped_name
+
+##31
+@R r31a :: base_type_spec >> floating_pt_type
+@R r31b :: base_type_spec >> integer_type
+@R r31c :: base_type_spec >> char_type
+@R r31d :: base_type_spec >> boolean_type
+@R r31e :: base_type_spec >> octet_type
+@R r31f :: base_type_spec >> any_type
+
+## 32
+@R r32a :: template_type_spec >> sequence_type
+@R r32b :: template_type_spec >> string_type
+
+##33
+@R r33a :: constr_type_spec >> struct_type
+@R r33b :: constr_type_spec >> union_type
+@R r33c :: constr_type_spec >> enum_type
+
+##34
+@R r34a :: declarators >> declarator
+@R r34b :: declarators >> declarators , declarator
+
+##35
+@R r35a :: declarator >> simple_declarator
+@R r35b :: declarator >> complex_declarator
+
+##36
+@R r36 :: simple_declarator >> identifier
+
+##37
+@R r37 :: complex_declarator >> array_declarator
+
+##38 kw float double
+@R r38a :: floating_pt_type >> float
+@R r38b :: floating_pt_type >> double
+
+##39
+@R r39a :: integer_type >> signed_int
+@R r39b :: integer_type >> unsigned_int
+
+##40
+@R r40 :: signed_int >> signed_long_int
+@R r40 :: signed_int >> signed_short_int
+
+##41 kw long
+@R r41 :: signed_long_int >> long
+
+##42 kw short
+@R r42 :: signed_short_int >> short
+
+##43
+@R r43 :: unsigned_int >> unsigned_long_int
+@R r43 :: unsigned_int >> unsigned_short_int
+
+##44 kw unsigned
+@R r44 :: unsigned_long_int >> unsigned long
+
+##45
+@R r45 :: unsigned_short_int >> unsigned short
+
+##46 kw char
+@R r46 :: char_type >> char
+
+##47 kw boolean
+@R r47 :: boolean_type >> boolean
+
+##48 kw octet
+@R r48 :: octet_type >> octet
+
+##49 kw any
+@R r49 :: any_type >> any
+
+##50 kw struct
+@R r50 :: struct_type >> struct identifier { member_list }
+
+##51
+@R r51a :: member_list >> member
+@R r51b :: member_list >> member_list member
+
+##52
+@R r52 :: member >> type_spec declarators ;
+
+##53 kw union switch
+@R r53 :: union_type >>
+ union identifier switch ( switch_type_spec ) { switch_body }
+
+##54
+@R r54a :: switch_type_spec >> integer_type
+@R r54b :: switch_type_spec >> char_type
+@R r54c :: switch_type_spec >> boolean_type
+@R r54d :: switch_type_spec >> enum_type
+@R r54e :: switch_type_spec >> scoped_name
+
+##55
+@R r55a :: switch_body >> case_nt
+@R r55b :: switch_body >> switch_body case_nt
+
+##56
+@R r56a :: case_nt >> case_labels element_spec ;
+@R r56b :: case_labels >> case_label
+@R r56c :: case_labels >> case_labels case_label
+
+
+##57 kw default case
+@R r57a :: case_label >> case const_expr :
+@R r57b :: case_label >> default :
+
+##58
+@R r58 :: element_spec >> type_spec declarator
+
+##59 kw enum
+@R r59a :: enum_type >> enum identifier { enumerators }
+@R r59b :: enumerators >> enumerator
+@R r59c :: enumerators >> enumerators , enumerator
+
+##60
+@R r60 :: enumerator >> identifier
+
+##61 kw sequence
+@R r61 :: sequence_type >> sequence < simple_type_spec , positive_int_const >
+
+##62 kw string
+@R r62a :: string_type >> string < positive_int_const >
+@R r62b :: string_type >> string
+
+##63
+@R r63a :: array_declarator >> identifier fixed_array_sizes
+@R r63b :: fixed_array_sizes >> fixed_array_size
+@R r63c :: fixed_array_sizes >> fixed_array_sizes fixed_array_size
+
+##64
+@R r64 :: fixed_array_size >> [ positive_int_const ]
+
+##65 kw attribute readonly
+@R r65a :: attr_dcl >> maybe_readonly attribute param_type_spec simple_declarators
+@R r65b :: maybe_readonly >> readonly
+@R r65c :: maybe_readonly >>
+@R r65d :: simple_declarators >> simple_declarator
+@R r65e :: simple_declarators >> simple_declarators , simple_declarator
+
+##66 kw exception
+@R r66a :: except_dcl >> exception identifier { members }
+@R r66b :: members >>
+@R r66c :: members >> member_list
+
+##67
+@R r67a :: op_dcl >>
+ maybe_op_attribute op_type_spec identifier parameter_dcls
+ maybe_raises_expr maybe_context_expr
+@R r67b :: maybe_op_attribute >>
+@R r67c :: maybe_op_attribute >> op_attribute
+@R r67d :: maybe_raises_expr >>
+@R r67e :: maybe_raises_expr >> raises_expr
+@R r67f :: maybe_context_expr >>
+@R r67g :: maybe_context_expr >> context_expr
+
+##68 kw oneway
+@R r68a :: op_attribute >> oneway
+
+##69 kw void
+@R r69a :: op_type_spec >> param_type_spec
+@R r69b :: op_type_spec >> void
+
+##70
+@R r70a :: parameter_dcls >> ( parameterlist )
+@R r70b :: parameter_dcls >> ( )
+@R r70c :: parameterlist >> param_dcl
+@R r70d :: parameterlist >> parameterlist , param_dcl
+
+##71
+@R r71 :: param_dcl >> param_attribute param_type_spec simple_declarator
+
+##72 kw in out inout
+@R r72 :: param_attribute >> in
+@R r72 :: param_attribute >> out
+@R r72 :: param_attribute >> inout
+
+##73 kw raises
+@R r73 :: raises_expr >> raises ( scoped_name_list )
+
+##74 kw context
+@R r74 :: context_expr >> context ( string_literal_list )
+@R r74b :: string_literal_list >> string_literal
+@R r74c :: string_literal_list >> string_literal_list , string_literal
+
+@R r75 :: param_type_spec >> base_type_spec
+@R r75 :: param_type_spec >> string_type
+@R r75 :: param_type_spec >> scoped_name
+
+"""
+
+nonterms = """
+colon_colon
+param_attribute
+unsigned_long_int unsigned_short_int param_dcl
+parameterlist string_literal_list
+members maybe_op_attribute maybe_raises_expr maybe_context_expr
+op_type_spec parameter_dcls op_attribute raises_expr context_expr
+maybe_readonly param_type_spec simple_declarators simple_declarator
+fixed_array_sizes fixed_array_size
+element_spec enumerator enumerators
+switch_type_spec switch_body case_nt case_labels case_label
+member_list member
+signed_int unsigned_int signed_long_int signed_short_int
+simple_declarator complex_declarator array_declarator
+declarator
+sequence_type string_type
+floating_pt_type integer_type char_type boolean_type
+octet_type any_type
+base_type_spec template_type_spec
+simple_type_spec constr_type_spec
+type_spec declarators
+type_declarator struct_type union_type enum_type
+literal boolean_literal positive_int_literal
+mult_expr unary_expr unary_operator primary_expr
+or_expr xor_expr and_expr shift_expr add_expr
+integer_type char_type boolean_type floating_type string_type
+const_type const_expr
+scoped_name_list scoped_name
+attr_dcl op_dcl
+inheritance_spec export
+interface_header interface_body
+interface_dcl forward_dcl
+type_dcl const_dcl except_dcl interface_nt module_nt
+specification definition speclist
+"""
+
+keywords = """
+exception oneway void in out inout raises context
+interface module const TRUE FALSE typedef float double long
+unsigned short char boolean octet any struct union switch
+enum string attribute readonly default case sequence ::
+"""
+# NOTE: FOR NECESSARY HACKERY REASONS :: IS A KEYWORD!
+
+punctuations = ";{}()[],:|^&<>+-*/%~="
+
+# dummy regexen
+identifierre = "identifier"
+integer_literalre = "123"
+positive_int_constre = "999"
+string_literalre = "'string'"
+character_literalre= "'c'"
+floating_pt_literalre = "1.23"
+
+
+# dummy interp fun for all terminals
+def echo (str):
+ return str
+
+def DeclareTerminals(Grammar):
+ Grammar.Addterm("identifier", identifierre, echo)
+ Grammar.Addterm("integer_literal", integer_literalre, echo)
+ Grammar.Addterm("string_literal", string_literalre, echo)
+ Grammar.Addterm("character_literal", character_literalre, echo)
+ Grammar.Addterm("floating_pt_literal", floating_pt_literalre, echo)
+ Grammar.Addterm("positive_int_const", positive_int_constre, echo)
+
+## we need to override LexDictionary to recognize :: as a SINGLE punctuation.
+## (not possible using standard kjParsing, requires a special override)
+import kjParser
+class myLexDictionary(kjParser.LexDictionary):
+ def __init__(self):
+ kjParser.LexDictionary.__init__(self)
+ map = ((kjParser.KEYFLAG, "coloncolon"), "coloncolon")
+ self.keywordmap["::"] = map
+ self.keywordmap["coloncolon"] = map
+
+ def Token(self, String, StartPosition):
+ if String[StartPosition:StartPosition+2] == "::":
+ tok = self.keywordmap["::"]
+ return (tok, 2)
+ # default:
+ return kjParseBuild.LexDictionary.Token(self, String, StartPosition)
+
+# default bind all rules
+
+def GrammarBuild():
+ import kjParseBuild
+ idl = kjParseBuild.NullCGrammar()
+ idl.LexD = myLexDictionary()
+ #idl.SetCaseSensitivity(0) # grammar is not case sensitive for keywords
+ DeclareTerminals(idl)
+ idl.Keywords(keywords)
+ idl.punct(punctuations)
+ idl.Nonterms(nonterms)
+ #idl.comments([LISPCOMMENTREGEX])
+ idl.Declarerules(idlgramstring)
+ print "now compiling"
+ idl.Compile()
+ return idl
+
+if __name__=="__main__": GrammarBuild()
=== Zope3/src/zope/app/rdb/gadfly/index.html 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/index.html Wed Dec 25 09:13:13 2002
@@ -0,0 +1,239 @@
+<HTML>
+<HEAD>
+<TITLE>
+Gadfly/kwParsing Downloads and Documentation
+</TITLE>
+
+</HEAD>
+
+<BODY bgcolor="#ffffff">
+<table>
+<tr>
+<td valign="top"><img src="gadfly.JPG"></td>
+<td>
+<h1>Gadfly 1.0: An SQL Database in Python</h1>
+Gadfly/kwParsing downloads and documentation.
+<p>
+<b>Gadfly requires the kwParsing parser/parser generator package.
+ The "two packages" are released as one distribution (kwP) with the
+ same generous copyright.
+<p>
+<a href="kwP.tar.gz">Download kwP.tar.gz in tar-gz format with tar.gz extension.</a>
+<p>
+<a href="kwP.tgz">Download kwP.tgz in tar-gz format with tgz extension.</a>
+<p>
+<a href="kjbuckets.pyd">Separately download kjbuckets.pyd data structure
+accellerator extension module for Python 1.5 on Windows NT or Windows 95/98
+only.</a>
+</b>
+<p>
+[The tar.gz extension doesn't always work for MSIE browsers.]
+<p>
+<a href="gffaq.html">Gadfly frequently asked questions (FAQ).</a>
+<p>
+<a href="gadfly.html">General information with links to other documentation,
+<em>including installation instructions</em>.</a>
+</td>
+</tr></table>
+<hr><br>
+<table BORDER>
+<tr>
+<td width="40%">
+
+ <a href="gadfly.py">gadfly.py</a> main module<br>
+ <a href="gfserve.py">gfserve.py</a> main TCP/IP server mode<br>
+ <a href="gfclient.py">gfclient.py</a> main TCP/IP client mode<br>
+ <a href="gfsocket.py">gfsocket.py</a> client/server support module<br>
+ <a href="gfstest.py">gfstest.py</a> client/server test module<br>
+ <a href="gfinstall.py">gfinstall.py</a> installation script<br>
+ <a href="gftest.py">gftest.py</a> test suite<br>
+ <a href="sqlbind.py">sqlbind.py</a> grammar bindings<br>
+ <a href="sqlgen.py">sqlgen.py</a> grammar generation<br>
+ <a href="sqlgram.py">sqlgram.py</a> SQL grammar<br>
+ <a href="sqlgtest.py">sqlgtest.py</a> grammar tests<br>
+ <a href="sqlsem.py">sqlsem.py</a> main semantic objects<br>
+ <a href="sqlmod.py">sqlmod.py</a> update semantic objects<br>
+ <a href="gfdb0.py">gfdb0.py</a> data and file archiving<br>
+ <a href="gfintrospect.py">gfintrospect.py</a> introspection and remote table support<br>
+ <a href="remotetest.py">remotetest.py</a> remote table implementation demo and test</a><br>
+ <a href="relalg.py">relalg.py</a> relational algebra interpreter (toy)<br>
+ <a href="kjbuckets0.py">kjbuckets0.py</a> base data structures
+(python version)<br>
+
+</td>
+<td valign="top">
+These are the core files to the
+<a href="gadfly.html">
+Gadfly SQL database engine
+</a>
+A relational database query engine that supports
+the Structured Query Language (SQL), implemented entirely
+in Python (with optional builtin support from the
+<a href="../kjbuckets">
+kjbuckets builtin data structure accelerator</a>).
+<hr>
+Gadfly has been tested on Windows 95, Windows NT, Linux, and Unix
+(solaris),
+and it should run anywhere that Python runs (bebox,
+maybe Palm Pilot/WinCE eventually...)
+<hr>
+Supports <a href="gfrecover.html">transactions and recovery</a>
+<hr>
+Supports <a href="gfSQL.html">a LARGE subset of SQL</a>
+<hr>
+Supports <a href="server.html">client/server access
+via TCP/IP sockets.</a>
+<hr>
+<em>Many thanks to users who have used Gadfly in previous
+ releases and who have offered suggestions and bug reports!</em>
+
+</td>
+</tr>
+<caption align="top"><b>Gadfly source modules and documentation</b></caption>
+</table>
+<h2>kwParser, Python lint, IDL, etc...</h2>
+
+kwParser is a parser generator for Python. It transforms an
+abstract specification of a language grammar (for example the
+CORBA Interface Definition Language) together with "interpretation
+functions" that define the semantics of the language into a
+compiler or translator or interpreter. In the case of CORBA IDL
+a python program using kwParser could generate stubs and support
+code (in Python or some other language) to talk to a CORBA interface.
+<p>
+The release given here has had some micro-optimizations (26 June 1997)
+which with luck don't break anything. In particular regexen are
+used more intelligently and the generation phase will use
+<a href="../kjbuckets">kjbuckets</a>
+if it's available.
+</p>
+This is moderately heavy computer science. Not for the timid.
+<p>
+<TABLE BORDER>
+ <TR>
+
+ <TD><a href="COPYRIGHT">COPYRIGHT</a>
+ </TD>
+ <td>Do what you like, just don't sue me (roughly translated).
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="DLispShort.py">DLispShort.py</a>
+ </TD>
+ <td>A very simple example language specification.
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="DumbLispGen.py">DumbLispGen.py</a>
+ </TD>
+ <td>Another example specification
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="arefize.py">arefize.py</a>
+ </TD>
+ <td>(not related: simple program used to generate this page)
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="idl.py">idl.py</a>
+ </TD>
+ <td>A very complex example: CORBA IDL parser generator.
+ This is a good example of a complex grammar. The
+ interpretation functions and terminal regexes are
+ all stubbed.
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="pygram.py">pygram.py</a>
+ </TD>
+ <td>A very complex example: The python grammar.
+ This module uses a hand written lexer to handle
+ Python's beautiful peculiarities. Used by kypylint.py.<br>
+ Look at the top of the module for editable parameters.
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="kjpylint.py">kjpylint.py</a>
+ </TD>
+ <td>An attempt to use the pygram parser to do simple
+ checking on python source files. Reports references
+ not set, assignments not used, etcetera. not all warnings
+ indicate real problems of course. Used like this:
+<pre>
+% python kjpylint.py /home/app/arw/Python-1.5a1/Lib/SocketServer.py
+setup
+loading
+now parsing
+(verify_request) 'request' defined before 225 not used
+(verify_request) 'self' defined before 225 not used
+(verify_request) 'client_address' defined before 225 not used
+(handle_error) 'request' defined before 245 not used
+(handle_error) 'self' defined before 245 not used
+(collect_children) 'status' defined before 293 not used
+(setup) 'self' defined before 368 not used
+(__del__) 'self' defined before 371 not used
+(handle) 'self' defined before 374 not used
+(finish) 'self' defined before 377 not used
+(<module global>) '__version__' defined before 104 not used
+269: (qref) 'max_packet_size' not defined in module?
+</pre>
+ Here only the last line indicates a possible real bug
+ in SocketServer.py
+<hr>
+ <em>Bugs/Features: <br>
+ Barfs on inconsistent indentation (ie space-tab is not the same as tab).
+ <br>
+ Grumpy about one line for loops and lambdas.<br>
+ For very weird cases may not parse strings correctly.<br>
+ See top of module for more info.<br>
+ Right now kjpylint might like to see an extra newline
+ at the end of the file. I'll look into this, sorry.</em>
+ <br>
+ Latest: Thu Jul 17 13:50:03 EDT 1997
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="kjParseBuild.py">kjParseBuild.py</a>
+ </TD>
+ <td>The Parser generator module, used only to generate
+ the data structures required for parsing.
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="kjParser.py">kjParser.py</a>
+ </TD>
+ <td>The Parser module, used both during the generation
+ phase and also after generation when the generated
+ parser is used.
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="kwParsing.html">kwParsing.html</a>
+ </TD>
+ <td>HTML documentation for the package.
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="kjSet.py">kjSet.py</a>
+ </TD>
+ <td>support module for parser generation.
+ (uses <a href="../kjbuckets">kjbuckets</a>
+ builtin if available, or uses a straight
+ Python implementation if not.)
+ </td>
+ </TR>
+ <TR>
+ <TD><a href="kwP.tar.gz">kwP.tar.gz</a>
+ </TD>
+ <td>The package (tarred, gzipped)
+ </td>
+ </TR>
+ <CAPTION ALIGN="top">kwParsing parser generator related files.
+ </CAPTION>
+ </TABLE>
+
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a>
+</BODY>
+</HTML>
=== Zope3/src/zope/app/rdb/gadfly/kjParseBuild.py 1.1 => 1.2 === (1232/1332 lines abridged)
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/kjParseBuild.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,1329 @@
+#
+# python code for building a parser from a grammar
+# Copyright Aaron Robert Watters, 1994
+#
+
+# BUGS:
+# A bad grammar that has no derivations for
+# the root nonterminal may cause a name error
+# on the variable "GoodStartingPlace"
+
+# this needs to be modified so the RULEGRAM is loaded from a
+# compiled representation if available.
+
+import string
+import kjSet
+import kjParser
+import re
+
+# import some constants
+from kjParser import \
+ TERMFLAG, NOMATCHFLAG, MOVETOFLAG, REDUCEFLAG, TRANSFLAG, KEYFLAG, \
+ NONTERMFLAG, TERMFLAG, EOFFLAG, ENDOFFILETOKEN
+
+PMODULE = kjParser.THISMODULE
+
+# errors raised here
+TokenError = "TokenError" # may happen on autogen with bad grammar
+NotSLRError = "NotSLRError" # may happen for nonSLR grammar
+
+# set this flag for regression testing at each load
+RUNTESTS = 0
+# set this flag to abort automatic generation on Errors
+ABORTONERROR = 0
+
+# token used to mark null productions
+NULLTOKEN = (None,None)
+
+
+# a derived FSM class, with closure computation methods defined
+# (compilable FSMachine)
+#
+class CFSMachine(kjParser.FSMachine):
+
+ def __init__(self, nonterm):
+ kjParser.FSMachine.__init__(self, nonterm)
+
+ # return the epsilon closure of the FSM as a new FSM
[-=- -=- -=- 1232 lines omitted -=- -=- -=-]
+ RX = kjParser.ParseRule( X, [ oppar, Y, clpar ] )
+ RY = kjParser.ParseRule( Y, [] )
+ rl2 = [RX,RY]
+ rs2 = ruleset(X, rl2)
+ rs2.CompFirst()
+ rs2.CompFollow()
+ rs2.CompSLRNFA()
+ rs2.CompDFA()
+ rs2.SLRFixDFA()
+ DFA2 = rs2.DFA
+
+ ttt2 = dummy()
+ def TESTDFA2( STRING, DOREDUCTIONS = 1):
+ return TESTDFA( STRING, ttt2, DFA2, rl2, DOREDUCTIONS )
+
+ # the following grammar should fail to be slr
+ # (Aho,Ullman p. 213)
+
+ S = kjParser.nonterminal("S")
+ L = kjParser.nonterminal("L")
+ R = kjParser.nonterminal("R")
+ RS1 = kjParser.ParseRule( S, [L, equals, R] )
+ RS2 = kjParser.ParseRule( S, [R], echo )
+ RL1 = kjParser.ParseRule( L, [star, R])
+ RL2 = kjParser.ParseRule( L, [id])
+ RR1 = kjParser.ParseRule( R, [L] )
+ rs3 = ruleset(S, [RS1,RS2,RL1,RL2,RR1])
+ rs3.CompFirst()
+ rs3.CompFollow()
+ rs3.CompSLRNFA()
+ rs3.CompDFA()
+ #rs3.SLRFixDFA() # should fail and does.
+
+ # testing RULEGRAM
+ ObjG = NullCGrammar()
+ ObjG.Addterm("id","id",echo)
+ ObjG.Nonterms("T E Ep F Tp")
+ ObjG.Keywords("begin end")
+ ObjG.punct("+*()")
+ ObjG.comments(["--.*\n"])
+ # PROBLEM WITH COMMENTS???
+ Rulestr = """
+ ## what a silly grammar!
+ T ::
+ @R One :: T >> begin E end
+ @R Three :: E >>
+ @R Two :: E >> E + T
+ @R Four :: E >> ( T )
+ """
+ RL = RULEGRAM.DoParse1( Rulestr, ObjG )
=== Zope3/src/zope/app/rdb/gadfly/kjParser.py 1.1 => 1.2 === (1213/1313 lines abridged)
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/kjParser.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,1310 @@
+#
+# python for parser interpretation
+# Copyright Aaron Robert Watters, 1994
+#
+
+# BUGS:
+# Lexical error handling is not nice
+# Parse error handling is not nice
+#
+# Lex analysis may be slow for big grammars
+# Setting case sensitivity for keywords MUST happen BEFORE
+# declaration of keywords.
+
+import kjSet
+import string
+import re
+import string
+
+# set this flag for regression testing at each load
+RUNTESTS = 0
+
+# set this flag to enable warning for default reductions
+WARNONDEFAULTS = 0
+
+# some local constants
+TERMFLAG = -1 # FLAG FOR TERMINAL
+NOMATCHFLAG = -2 # FLAG FOR NO MATCH IN FSM
+MOVETOFLAG = -3 # FLAG FOR "SHIFT" IN SN FSM
+REDUCEFLAG = -4 # FLAG FOR REDUCTION IN FSM
+TRANSFLAG = -5 # FLAG FOR TRANSIENT STATE IN FSM
+KEYFLAG = -6 # FLAG FOR KEYWORD
+NONTERMFLAG = -7 # FLAG FOR NONTERMINAL
+TERMFLAG = -8 # FLAG FOR TERMINAL
+EOFFLAG = "*" # FLAG for End of file
+
+# set this string to the Module name (filename)
+# used for dumping reconstructable objects
+THISMODULE = "kjParser"
+
+# regular expression for matching whitespace
+WHITERE = "["+string.whitespace+"]+"
+WHITEREGEX = re.compile(WHITERE)
+
+# local errors
+LexTokenError = "LexTokenError" # may happen on bad string
+UnkTermError = "UnkTermError" # ditto
+BadPunctError= "BadPunctError" # if try to make whitespace a punct
[-=- -=- -=- 1213 lines omitted -=- -=- -=-]
+ for tokenindex in range(len(tokens)):
+ (kind,name) = tokens[tokenindex]
+ if kind == KEYFLAG:
+ tokens[tokenindex] = LexD.keyword(name)
+ elif not kind in [TERMFLAG, NONTERMFLAG]:
+ raise FlowError, "unknown token type"
+ # not needed
+ self.tokens = tokens
+
+ def MakeRules(self):
+ Grammar = self.Gram
+ Grammar.DFA.root_nonTerminal = self.Root
+ NameIndex = Grammar.RuleNameToIndex
+ RuleTuples = self.RuleTups
+ nRules = len(RuleTuples)
+ RuleList = [None] * nRules
+ for index in range(nRules):
+ (Name, Components) = RuleTuples[index]
+ rule = apply(ParseRule, Components)
+ rule.Name = Name
+ RuleList[index] = rule
+ NameIndex[Name] = index
+ Grammar.RuleL = RuleList
+
+ def MakeTransitions(self):
+ Grammar = self.Gram
+ DFA = Grammar.DFA
+ StateTokenMap = DFA.StateTokenMap
+ tokens = self.tokens
+ # record the state number
+ DFA.maxState = self.MaxStates
+ # this is historical, unfortunately... CLEAN IT UP SOMEDAY!
+ # THE DFA.States DICT IS NOT NEEDED (?) (here)
+ for state in range(1, self.MaxStates+1):
+ DFA.States[state] = [TRANSFLAG]
+ # record the reductions
+ for (fromState, TokenIndex, rulenum) in self.reducts:
+ DFA.SetReduction(fromState, tokens[TokenIndex], rulenum)
+ # record the transitions
+ for (fromState, TokenIndex, ToState) in self.moveTos:
+ DFA.SetMap(fromState, tokens[TokenIndex], ToState)
+
+ def Cleanup(self):
+ Grammar = self.Gram
+ Grammar.CleanUp()
+
+################# FOLLOWING CODE IS FOR REGRESSION TESTING ONLY
+################# DELETE IT IF YOU WANT/NEED
+#### All tests for this module deleted, since
+#### ParseBuild module tests are sufficient.
=== Zope3/src/zope/app/rdb/gadfly/kjSet.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/kjSet.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,254 @@
+
+#
+# sets implemented using mappings
+# Copyright Aaron Robert Watters, 1994
+#
+# these only work for "immutable" elements.
+# probably not terribly efficient, but easy to implement
+# and not as slow as concievably possible.
+
+def NewSet(Sequence):
+ Result = {}
+ for Elt in Sequence:
+ Result[Elt] = 1
+ return Result
+
+def Empty(Set):
+ if Set == {}:
+ return 1
+ else:
+ return 0
+
+def get_elts(Set):
+ return Set.keys()
+
+def member(Elt,Set):
+ return Set.has_key(Elt)
+
+# in place mutators:
+# returns if no change otherwise 1
+
+def addMember(Elt,Set):
+ change = 0
+ if not Set.has_key(Elt):
+ Set[Elt] = 1
+ change = 1
+ return change
+
+def Augment(Set, OtherSet):
+ change = 0
+ for Elt in OtherSet.keys():
+ if not Set.has_key(Elt):
+ Set[Elt] = 1
+ change = 1
+ return change
+
+
+def Mask(Set, OtherSet):
+ change = 0
+ for Elt in OtherSet.keys():
+ if Set.has_key(Elt):
+ del Set[Elt]
+ change = 1
+ return change
+
+# side effect free functions
+
+def Intersection(Set1, Set2):
+ Result = {}
+ for Elt in Set1.keys():
+ if Set2.has_key(Elt):
+ Result[Elt] = 1
+ return Result
+
+def Difference(Set1, Set2):
+ Result = {}
+ for Elt in Set1.keys():
+ if not Set2.has_key(Elt):
+ Result[Elt] = 1
+ return Result
+
+def Union(Set1,Set2):
+ Result = {}
+ Augment(Result,Set1)
+ Augment(Result,Set2)
+ return Result
+
+def Subset(Set1,Set2):
+ Result = 1
+ for Elt in Set1.keys():
+ if not Set2.has_key(Elt):
+ Result = 0
+ return Result # nonlocal
+ return Result
+
+def Same(Set1,Set2):
+ if Subset(Set1,Set2) and Subset(Set2,Set1):
+ return 1
+ else:
+ return 0
+
+# directed graphs as Dictionaries of Sets
+# also only works for immutable nodes
+
+def NewDG(pairlist):
+ Result = {}
+ for (source,dest) in pairlist:
+ AddArc(Result, source, dest)
+ return Result
+
+def GetPairs(Graph):
+ result = []
+ Sources = Graph.keys()
+ for S in Sources:
+ Dests = get_elts( Graph[S] )
+ ThesePairs = [None] * len(Dests)
+ for i in range(0,len(Dests)):
+ D = Dests[i]
+ ThesePairs[i] = (S, D)
+ result = result + ThesePairs
+ return result
+
+def AddArc(Graph, Source, Dest):
+ change = 0
+ if Graph.has_key(Source):
+ Adjacent = Graph[Source]
+ if not member(Dest,Adjacent):
+ addMember(Dest,Adjacent)
+ change = 1
+ else:
+ Graph[Source] = NewSet( [ Dest ] )
+ change = 1
+ return change
+
+def Neighbors(Graph,Source):
+ if Graph.has_key(Source):
+ return get_elts(Graph[Source])
+ else:
+ return []
+
+def HasArc(Graph, Source, Dest):
+ result = 0
+ if Graph.has_key(Source) and member(Dest, Graph[Source]):
+ result = 1
+ return result
+
+def Sources(Graph):
+ return Graph.keys()
+
+# when G1, G2 and G3 are different graphs this results in
+# G1 = G1 U ( G2 o G3 )
+# If G1 is identical to one of G2,G3 the result is somewhat
+# nondeterministic (depends on dictionary implementation).
+# However, guaranteed that AddComposition(G,G,G) returns
+# G1 U (G1 o G1) <= G <= TC(G1)
+# where G1 is G's original value and TC(G1) is its transitive closure
+# hence this function can be used for brute force transitive closure
+#
+def AddComposition(G1, G2, G3):
+ change = 0
+ for G2Source in Sources(G2):
+ for Middle in Neighbors(G2,G2Source):
+ for G3Dest in Neighbors(G3, Middle):
+ if not HasArc(G1, G2Source, G3Dest):
+ change = 1
+ AddArc(G1, G2Source, G3Dest)
+ return change
+
+# in place transitive closure of a graph
+def TransClose(Graph):
+ change = AddComposition(Graph, Graph, Graph)
+ somechange = change
+ while change:
+ change = AddComposition(Graph, Graph, Graph)
+ if not somechange:
+ somechange = change
+ return somechange
+
+########### SQueue stuff
+#
+# A GrabBag should be used to hold objects temporarily for future
+# use. You can put things in and take them out, with autodelete
+# that's all!
+
+# make a new baggy with nothing in it
+# BG[0] is insert cursor BG[1] is delete cursor, others are elts
+#
+OLD = 1
+NEW = 0
+START = 2
+def NewBG():
+ B = [None]*8 #default size
+ B[OLD] = START
+ B[NEW] = START
+ return B
+
+def BGempty(B):
+ # other ops must maintain this: old == new iff empty
+ return B[OLD] == B[NEW]
+
+# may return new, larger structure
+# must be used with assignment... B = BGadd(e,B)
+def BGadd(elt, B):
+ cursor = B[NEW]
+ oldlen = len(B)
+ # look for an available position
+ while B[cursor] != None:
+ cursor = cursor+1
+ if cursor >= oldlen: cursor = START
+ if cursor == B[NEW]: #back to beginning
+ break
+ # resize if wrapped
+ if B[cursor] != None:
+ B = B + [None] * oldlen
+ cursor = oldlen
+ B[OLD] = START
+ if B[cursor] != None:
+ raise IndexError, "can't insert?"
+ # add the elt
+ B[cursor] = (elt,)
+ B[NEW] = cursor
+ # B nonempty so OLD and NEW should differ.
+ if B[OLD] == cursor:
+ B[NEW] = cursor + 1
+ if B[NEW]<=len(B): B[NEW] = START
+ return B
+
+def BGgetdel(B):
+ # find something to delete:
+ cursor = B[OLD]
+ blen = len(B)
+ while B[cursor]==None:
+ cursor = cursor+1
+ if cursor>=blen: cursor = START
+ if cursor == B[OLD]: break # wrapped
+ if B[cursor] == None:
+ raise IndexError, "delete from empty grabbag(?)"
+ # test to see if bag is empty (position cursor2 at nonempty slot)
+ cursor2 = cursor+1
+ if cursor2>=blen: cursor2 = START
+ while B[cursor2]==None:
+ cursor2 = cursor2+1
+ if cursor2>=blen: cursor2 = START
+ # since B[cursor] not yet deleted while will terminate
+ # get and delete the elt
+ (result,) = B[cursor]
+ B[cursor] = None
+ # cursor == cursor2 iff bag is empty
+ B[OLD] = cursor2
+ if B[NEW] == cursor2: B[NEW] = cursor
+ return result
+
+def BGtest(n):
+ B = NewBG()
+ rn = range(n)
+ rn2 = range(n-2)
+ for i in rn:
+ for j in rn:
+ B = BGadd( (i,j), B)
+ B = BGadd( (j,i), B)
+ x = BGgetdel(B)
+ for j in rn2:
+ y = BGgetdel(B)
+ print (i, x, y)
+ return B
=== Zope3/src/zope/app/rdb/gadfly/kjbuckets0.py 1.1 => 1.2 === (948/1048 lines abridged)
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/kjbuckets0.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,1045 @@
+
+# kjbuckets in pure python
+
+### needs more thorough testing!
+
+#import sys # for debug
+
+def kjtabletest(x):
+ #print "kjtabletest"
+ try:
+ return x.is_kjtable
+ except:
+ return 0
+
+unhashable = "unhashable key error"
+
+class kjGraph:
+
+ is_kjtable = 1
+
+ def __init__(self, *args):
+ #print "kjGraph.__init__", args
+ key_to_list = self.key_to_list = {}
+ self.dirty = 0
+ self.hashed = None
+ #print args
+ if args:
+ if len(args)>1:
+ raise ValueError, "only 1 or 0 argument supported"
+ from types import IntType, ListType, TupleType
+ arg = args[0]
+ targ = type(arg)
+ test = key_to_list.has_key
+ if type(arg) is IntType:
+ return # ignore int initializer (presize not implemented)
+ elif type(arg) is ListType or type(arg) is TupleType:
+ for (x,y) in arg:
+ if test(x):
+ key_to_list[x].append(y)
+ else:
+ key_to_list[x] = [y]
+ return
+ aclass = arg.__class__
+ if aclass is kjGraph:
+ aktl = arg.key_to_list
+ for k in aktl.keys():
+ key_to_list[k] = aktl[k][:]
[-=- -=- -=- 948 lines omitted -=- -=- -=-]
+ del X[2]
+ print X, "Clean after", X.Clean()
+ if not X.subset(X):
+ raise "trivial subset fails", X
+ if not X==X:
+ raise "trivial cmp fails", X
+ if not X:
+ raise "nonzero fails", X
+ if X is S:
+ if not S.member(0):
+ raise "huh 1?"
+ if S.member(123):
+ raise "huh 2?", S
+ S.add(999)
+ del S[1]
+ if not S.has_key(999):
+ raise "huh 3?", S
+ else:
+ print "values", X.values()
+ print "keys", X.keys()
+ print X, "inverted", ~X
+ if not X.member(0,1):
+ raise "member test fails (0,1)", X
+ print "adding to", X
+ X.add(999,888)
+ print "added", X
+ X.delete_arc(999,888)
+ print "deleted", X
+ if X.member(999,888):
+ raise "member test fails (999,888)", X
+ if X.has_key(999):
+ raise "has_key fails 999", X
+ if not X.has_key(0):
+ raise "has_key fails 0", X
+ for Y in ALL:
+ print "Y", Y
+ if (X!=S and Y!=S):
+ print "diff", X, Y
+ print "%s-%s=%s" % (X,Y,X-Y)
+ elif X==S:
+ D = kjSet(Y)
+ print "diff", X, D
+ print "%s-%s=%s" % (X,D,X-D)
+ print "%s+%s=%s" % (X,Y,X+Y)
+ print "%s&%s=%s" % (X,Y,X&Y)
+ print "%s*%s=%s" % (X,Y,X*Y)
+ x,y = cmp(X,Y), cmp(Y,X)
+ if x!=-y: raise "bad cmp!", (X, Y)
+ print "cmp(X,Y), -cmp(Y,X)", x,-y
+ print "X.subset(Y)", X.subset(Y)
=== Zope3/src/zope/app/rdb/gadfly/kjpylint.py 1.1 => 1.2 === (503/603 lines abridged)
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/kjpylint.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,600 @@
+#!/usr/local/bin/python
+"""python lint using kwParsing
+
+The goal of this module/filter is to help find
+programming errors in python source files.
+
+As a filter use thusly:
+
+% python kjpylint.py source_file.py
+
+As an internal tool use like this:
+
+ import kjpylint
+ (pyg, context) = kjpylint.setup()
+ kjpylint.lint(data, pyg, context)
+
+where data is the text of a python program.
+You can build your own context structure by
+subclassing GlobalContext, and redefining
+GlobalContext.complain(string) for example.
+You could do a lot more than that too...
+
+Also, to lint all *.py files recursively contained
+in a directory hierarchy use
+
+ kjpylint.lintdir("/usr/local/lib/python") # for example
+
+FEATURES:
+
+Lint expects
+ 1) a newline or two at the end of the data;
+ 2) consistent indenting (and inconsistency may be invisible)
+ [eg " \t" and "\t" are not the same indent
+ to Lint, but Python sees them the same.]
+
+If (1) or (2) are not satisfied Lint will raise
+an exception.
+
+Buglets: lambdas and for loops on one line generate
+ extraneous warnings.
+
+Notes:
+======
+The lint process works, in outline, like this.
+Scan over a python program
+
+x = 1
[-=- -=- -=- 503 lines omitted -=- -=- -=-]
+class x(y,z):
+ '''
+ a doc string
+ blah
+ '''
+ def test(this, that):
+ w = that+this+x, n
+ x = 1
+ return w
+"""
+
+def go():
+ import sys
+ try:
+ file = sys.argv[1]
+ except IndexError:
+ print "required input file missing, defaulting to test string"
+ data = teststring
+ else:
+ data = open(file).read()
+ print "setup"
+ (pyg, context) = setup()
+ print "now parsing"
+ lint(data, pyg, context)
+
+def setup():
+ global pyg, context
+ import pygram
+ pyg = pygram.unMarshalpygram()
+ BindRules(pyg)
+ context = globalContext(pyg.LexD)
+ return (pyg, context)
+
+def lint(data, pygin=None, contextin=None):
+ if pygin is None: pygin = pyg
+ if contextin is None: contextin = context
+ pygin.DoParse1(data, contextin)
+
+def lintdir(directory_name):
+ """lint all files recursively in directory"""
+ from find import find
+ print "\n\nrecursively linting %s\n\n" % directory_name
+ (pyg, context) = setup()
+ python_files = find("*.py", directory_name)
+ for x in python_files:
+ print "\n\n [ %s ]\n\n" % x
+ lint( open(x).read(), pyg, context )
+ print "\014"
+
+if __name__=="__main__": go()
=== Zope3/src/zope/app/rdb/gadfly/kwParsing.html 1.1 => 1.2 === (989/1089 lines abridged)
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/kwParsing.html Wed Dec 25 09:13:13 2002
@@ -0,0 +1,1085 @@
+<html>
+<head>
+<title>A parser generator in Python</title>
+</head>
+<body bgcolor="#ffffff">
+<center>
+<h1>A parser generator in Python: kwParsing</h1>
+
+
+<blockquote>
+Aaron Watters<br>
+</blockquote>
+
+<blockquote>
+This is the documentation for the <strong>kjParsing</strong> package,
+an experimental parser generator implemented in Python which generates
+parsers implemented in Python.
+It won't serve as a complete reference on programming
+language syntax and interpretation, but it will review
+terminology for the knowledgable and I hope it will pique
+the interest of the less experienced.
+</blockquote>
+</center>
+
+<h2>Introduction</h2>
+<p>
+The <code> kjParsing</code> package is a parser generator written
+in Python which generates parsers for use in Python.
+<p>
+These modules and their documentation and demo files
+may be of use for classes on parsing, compiling, or
+formal languages, and may also be helpful to people
+who like to create experimental interpreters or translators
+or compilers.
+<p>
+The package consists of three Python modules:
+<code> kjParser, kjParseBuild,</code> and <code> kjSet</code>. Together these
+modules are called the <code> kjParsing</code> package.
+The package also includes some documentation and demo
+files and a <code> COPYRIGHT</code> file which explains the
+conditions for copying and propagating this code
+and the fact that the author assumes no responsibility
+for any difficulties resulting from the use of this
+package by anyone (including himself).
+
+<h2>What a Parser Does</h2>
+
[-=- -=- -=- 989 lines omitted -=- -=- -=-]
+the parser generator to prefer one binding or the other.
+No method for providing a preference is implemented here, yet.
+Let me know if you need such a method or if you have any suggestions.
+<p>
+Keywords of the meta-grammar cannot name tokens of
+the object grammar (see footnote above).
+<p>
+If you want keywords to be recognized without case
+sensitivity you must declare <code> G.SetCaseSensitivity(0)</code>
+before any keyword declarations.
+<p>
+Name and regular expression collisions are not always
+checked and reported. If you name two rules the same,
+for example, you may get undefined behavior.
+<p>
+The lexical analysis implementation is not as fast as it
+could be (of course).
+It also sees all white space as a
+`single space'
+so, for example, if indentation is significant in your grammar
+(as in Python) you'll need a different lexical analyzer.
+Also if <code> x=+y</code> means something different from
+<code> x = + y</code> (as it did in the original C, I believe)
+you may have trouble. Happily the lexical component can
+be easily ``plug replaced'' by another implementation if needed.
+<p>
+Also, the system currently only handles SLR grammars (as defined
+by Aho and Ullman), as mentioned above. If you get a
+<code> NonSLRError</code> during grammar compilation you need a better
+parser generator. I may provide one, if I have motivation and time.
+<p>
+I know of no outright bugs. Trust me, they're there. Please
+find them for me and tell me about them. I'm not a big
+expert on parsing so I'm sure I've made some errors, particularly
+at the lexical level.
+
+<h2>Further Reading</h2>
+<p>
+A standard reference for parsing and compiler, interpreter,
+and translator implementation is Principles of Compiler
+Design, by Aho and Ullman (Addison Wesley).
+
+
+<p>
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+</body>
+</html>
\ No newline at end of file
=== Zope3/src/zope/app/rdb/gadfly/pygram.py 1.1 => 1.2 === (897/997 lines abridged)
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/pygram.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,994 @@
+# rules for python
+# based on grammar given in Programming Python by Mark Lutz
+
+# EDIT THIS: THE DIRECTORY IN WHICH TO MARSHAL THE
+# GRAMMAR DATA STRUCTURES.
+#
+ARCHIVE = "."
+
+marshalfilename = ARCHIVE + "/pygram.mar"
+
+pyrules = """
+
+all ::
+
+## input terminates with "fake" dedent (forces read of all file)
+
+@R all1 :: all >> file_input DEDENT
+
+## 1 term newline
+
+##@R lead_blank :: file_input >> NEWLINE file_input
+
+@R top_stmt :: file_input >> file_input stmt
+@R file_input :: file_input >> stmt
+
+
+## 2
+@R simple :: stmt >> simple_stmt
+@R compound :: stmt >> compound_stmt
+
+## 3 punct ; term NEWLINE
+@R one_small :: simple_stmt >> small_stmt NEWLINE
+@R more_small :: simple_stmt >> small_stmt ; simple_stmt
+@R small_semi :: simple_stmt >> small_stmt ; NEWLINE
+
+## 4 kw pass
+@R smexpr :: small_stmt >> expr_stmt
+@R smassn :: small_stmt >> assn
+@R smprint :: small_stmt >> print_stmt
+@R smdel :: small_stmt >> del_stmt
+@R smpass :: small_stmt >> pass
+@R smflow :: small_stmt >> flow_stmt
+@R smimport :: small_stmt >> import_stmt
+@R smglobal :: small_stmt >> global_stmt
+## access ignored
+@R smexec :: small_stmt >> exec_stmt
+
[-=- -=- -=- 897 lines omitted -=- -=- -=-]
+ d = {}
+ for i in range(10): d[i] = i
+ '''
+ def test(c,s):
+ return "this"
+ while not done:
+ print done
+ break
+ list = [1,2,3]
+ # comment
+ return 5
+
+
+ n,x = 89 >> 90 + 6 / 7 % x + z << 6 + 2 ** 8
+
+if x==5:
+ while y:
+ for i in range(6):
+ raise SystemError, "oops"
+
+
+"""
+
+#teststring ="""\
+## comment
+#if x in y: print z
+#elif 1: print w
+#"""
+
+'''
+teststring="""
+exec "print 1"
+"""
+'''
+
+def test(grammar, context=None, teststring=teststring):
+ from time import time
+ now = time()
+ x = grammar.DoParse1(teststring, context)
+ elapsed = time()-now
+ print x
+ print elapsed
+ return x
+
+regen = 0
+dotest = 0
+
+if __name__ == "__main__" :
+ if regen: GrammarBuild()
+ unMarshalpygram()
=== Zope3/src/zope/app/rdb/gadfly/relalg.py 1.1 => 1.2 === (427/527 lines abridged)
--- /dev/null Wed Dec 25 09:13:46 2002
+++ Zope3/src/zope/app/rdb/gadfly/relalg.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,524 @@
+
+"""Simple relational algebra interpreter.
+
+usage:
+
+ To make the grammar
+
+ python relalg.py make
+
+ To run some relatoinal algebra expressions
+
+ python relalg.py < expressions_file
+
+"""
+
+# EDIT INSTALLDIR TO BE ABLE TO LOAD UNDER ANY CWD
+INSTALLDIR = "."
+
+## simple relational algebra using only the equality predicate
+## note: string values cannot contain ;
+
+## statement sequencing using ; handled at higher level
+
+relalg_rules = """
+
+statement ::
+
+@R statementassn :: statement >> assignment
+
+@R statementexpr :: statement >> rexpr
+
+@R assignment1 :: assignment >> name = rexpr
+
+@R assignmentn :: assignment >> name = assignment
+
+@R union :: rexpr >> rexpr U rterm
+
+@R rterm :: rexpr >> rterm
+
+@R minus :: rexpr >> rexpr - rterm
+
+@R intersect :: rterm >> rterm intersect rfactor
+
+@R join :: rterm >> rterm join rfactor
+
+@R rfactor :: rterm >> rfactor
+
[-=- -=- -=- 427 lines omitted -=- -=- -=-]
+ outfile.close()
+ return SQLG
+
+def reloadrelalg(filename=MARSHALFILE):
+ import kjParser
+ filename = INSTALLDIR+"/"+filename
+ infile = open(filename, "rb")
+ SQLG = kjParser.UnMarshalGram(infile)
+ infile.close()
+ DeclareTerminals(SQLG)
+ BindRules(SQLG)
+ return SQLG
+
+def runfile(f):
+ from string import split, join
+ ragram = reloadrelalg()
+ context = {}
+ #f = open(filename, "r")
+ data = f.read()
+ #f.close()
+ from string import split, strip
+ commands = split(data, ";")
+ for c in commands:
+ if not strip(c): continue
+ print " COMMAND:"
+ data = str(c)
+ pdata = " "+join(split(c, "\n"), "\n ")
+ print pdata
+ test = ragram.DoParse1(c, context)
+ print
+
+# c:\python\python relalg.py ratest.txt
+
+if __name__=="__main__":
+ try:
+ done = 0
+ import sys
+ argv = sys.argv
+ if len(argv)>1:
+ command = argv[1]
+ if command=="make":
+ print "building relational algebra grammar"
+ Buildrelalg()
+ done = 1
+ else:
+ runfile(sys.stdin)
+ done = 1
+ finally:
+ if not done:
+ print __doc__
=== Zope3/src/zope/app/rdb/gadfly/remotetest.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/remotetest.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,84 @@
+"""Demonstration of the Remote View protocol for adding
+ specially implemented Views in an application."""
+from gadfly import gadfly
+
+# create the database
+g = gadfly()
+g.startup("dbtest", "dbtest") # assume directory "dbtest" exists
+
+# define a remote view class
+import gfintrospect
+
+class myTable(gfintrospect.RemoteView):
+
+ """A remote view must define self.column_names
+ to return a (fixed) list of string column names and
+ self.listing() to return a possibly varying
+ list of row values. If there is a single column
+ the listing() list must return a list of values,
+ but for multiple columns it must return a list
+ of tuples with one entry for each column.
+
+ The remote view implementation may optionally
+ redefine __init__ also, please see gfintrospect.py
+ """
+
+ # static: don't reconstruct internal structure for each query
+ # for more interesting views static will generally be 0
+ static = 1
+
+ def __init__(self, column_names=None, rowlist=None):
+ """do whatever needed for initialization"""
+ if column_names is None:
+ column_names = ['a', 'b', 'c']
+ if rowlist is None:
+ rowlist = [(1,2,3), (4,5,6), (7,8,9)]
+ self.column_names = column_names
+ self.rowlist = rowlist
+
+ def listing(self):
+ """return list of tuples of right sizes to match column_names.
+ for more interesting views this will do something more
+ complex ;).
+ """
+ return self.rowlist
+
+# create a table using default cols and rows
+### Python code adding ANY remote views must be EXECUTED
+### EACH TIME THE DATABASE LOADS!
+
+g.add_remote_view("test", myTable())
+
+# create a table using specified cols and rows
+# NOTE: for single column give list of values
+# NOT list of tuples of values!
+g.add_remote_view("test2", myTable(["x"], [1,6,7]))
+
+print g.database
+
+c = g.cursor()
+
+c.execute("select * from test")
+print "test::"
+print c.pp()
+print
+c.execute("select * from test2")
+print "test2::"
+print c.pp()
+print
+c.execute("select * from test, test2 where x=a")
+print "join"
+print c.pp()
+print
+
+g.add_remote_view("test3", myTable(["z", "w"], [(2,3), (7,8), (4,6)]))
+c.execute("select * from test3")
+print "test3::"
+print c.pp()
+print
+
+c.execute(
+ "select * from test, test2, test3 where x=a and z=b and w=c")
+print "join 2::"
+print c.pp()
+print
=== Zope3/src/zope/app/rdb/gadfly/server.html 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/server.html Wed Dec 25 09:13:13 2002
@@ -0,0 +1,287 @@
+<html>
+<head>
+<title>Gadfly: server operations</title>
+</head>
+<body bgcolor="#ffffff">
+
+<table><tr><td>
+<img src="gadfly.JPG">
+</td><td>
+<h1>Gadfly Server Operations</h1>
+</td></tr></table>
+
+To permit multiple processes to access and modify a
+single database instance, and to reduce the overhead per process
+of connecting to a Gadfly database a Gadfly database may be
+run in server mode. A Gadfly server can use a DBA
+(data base administrator) configured
+start-up script to set up optimized query accesses and certain
+forms of security.
+<p>
+For example to startup a server for the
+test database "test" in directory "dbtest" (created by gftest.py)
+use:
+<pre>
+ python gfserve.py 2222 test dbtest admin
+</pre>
+or to start up the same server with some non-priviledged
+policies and some named prepared queries (as initialized
+in gfstest.startup(...)) use
+<pre>
+ python gfserve.py 2222 test dbtest admin gfstest
+</pre>
+In both cases the admin password for the server is "admin"
+and the server runs on port 2222.
+See the doc string for gfserve.py for more information on
+the command line arguments.
+<p>
+Only one process should directly access a gadfly database at once
+(not mediated by a server),
+so if a server is running, no other server for that database
+should be started and no other process should connect in "non-server"
+mode to that database.
+
+<h1>Motivation</h1>
+
+There are several reasons to run a server: to allow multiple
+processes to access the same database; to allow password protected
+restricted access to the database by non-priviledged agents;
+and to permit faster access to the database
+by providing globally shared
+prepared statements. Using a server also eliminates the need
+to start up and load the database many times -- and startup
+time could be considerable if the database is large.
+<p>
+For example I imagine that simple Gadfly servers may be of
+use to implement database enabled CGI scripts, whereas the
+"non-server" Gadfly will only run with CGI scripts that do not modify
+the database, and the startup time for Gadfly might make those
+scripts unacceptibly slow if the database is large. Furthermore,
+by using the security features a Gadfly server could be configured
+to allow restricted data distribution across a network without
+compromising the integrity of the database.
+
+<h1>Security</h1>
+
+The primary goal of Gadfly server security is to prevent
+accidental or malicious destruction of a database.
+<p>
+Security is arbitrated by policies. Policies have passwords
+that are never transmitted in clear text. However, a "captured"
+command could potentially be repeated by a hostile program
+even without knowing the password. It is not a good idea to
+run admin or other unrestricted commands on a network that may
+have hostile parties sniffing the network. As with the rest
+of the system I provide no guarantees, but for many purposes
+the level of security provided may be acceptible. To be specific
+passwords are used to generate md5 certificates for all server
+accesses (please see gfsocket.py for implementation details).
+<p>
+A server always has
+an "admin" policy that is permitted to shutdown, restart, or
+force a checkpoint on the server. By default the admin
+policy also has the ability to run arbitrary SQL statements
+such as "drop table x". This ability can be disabled in
+a startup function if needed.
+<pre>
+admin_policy.general_queries=0
+</pre>
+<p>
+Other policies can be created that have very restricted access.
+For example the following startup function initializes two
+policies beyond the admin policy that can only access certain
+tables in specific ways (from gfstest.py):
+<pre>
+def startup(admin_policy, connection, Server_instance):
+ """example startup script.
+
+ add a policies test and test1 passwords same
+ test1 is allowed to query the likess table by name
+ test is allowed to update likes where drinker='nan'
+ also add prepared query dumpwork to admin_policy.
+ """
+ from gfserve import Policy
+ admin_policy["dumpwork"] = "select * from work"
+ test1 = Policy("test1", "test1", connection, queries=0)
+ test = Policy("test", "test", connection, queries=0)
+ test1["qlike"] = "select * from likes where drinker=?"
+ test["updatenan"] = """
+ update likes
+ set beer=?, perday=?
+ where drinker='nan'
+ """
+ test["getnan"] = """
+ select * from likes where drinker='nan'
+ """
+ return {"test": test, "test1": test1}
+</pre>
+Please see the doc string for gfserve.py for more information
+on creating startup functions.
+<p>
+A policy with queries disabled (queries=0) can only execute
+named queries. By using such policies a DBA can configure
+a server such that client programs can only read certain tables,
+can only update certain rows of certain tables in certain ways,
+and so forth.
+<p>
+Even policies with "unrestricted access" (queries=1)
+can provide performance benefits if they have associated
+named, prepared queries (like "dumpwork" above). At the moment
+the SQL parser slows down gadfly a bit, and prepared queries
+will only be parsed once for all clients. After the first
+access subsequent accesses may be noticably faster (10x faster
+in some cases), especially
+if the server has the kjbuckets builtin C module. However,
+with queries=1 the policy can execute any SQL statement.
+<p>
+<strong>NOTE: The server runs all accesses (once the complete
+message has been read from the network) serially -- there is
+no concurrent access permitted to
+a Gadfly instance at this time. For this
+reason a "large query" may cause the server to "freeze" and
+delay other accesses.
+</strong> Incomplete requests due to network delays or
+other problems will not freeze the server, however (sockets
+are polled using select.select).
+<p>
+<strong>NOTE: All server accesses run in "autocommit mode" at
+this time. A successful access automatically triggers a database
+commit (but an unsuccessful access will rollback).</strong>
+<p>
+As an optimization, however, checkpoints only occur occasionally,
+once per a given number of accesses, configurable by setting:
+<pre>
+Server_instance.check_loop = 100
+</pre>
+
+<h1>Start up</h1>
+
+Servers can be started from the command line using the gfserve.py
+script interpretation
+(as shown above)
+or using gfserve.Server(...) from another
+program. See the doc strings and source for gfserve.py and gfstest.py
+for more information.
+
+<h1>Shut down</h1>
+
+Servers can be shut down from the command line interpretation of
+gfclient.py or from another program using the gfclient(...) class
+shutdown() method, but only using the admin policy with the admin
+password. For example to shut down the server started above:
+<pre>
+python gfclient.py shutdown 2222 admin
+</pre>
+See the doc strings and source for gfserve.py
+and gfstest.py
+for more information.
+
+<h1>Client Access</h1>
+
+Client access to a gadfly server is similar to the normal
+Python DB-SIG DBAPI
+access to gadfly, except that it is sometimes faster and can
+potentially
+be run from any machine reachable on the network (if the client
+program knows the password).
+<p>
+To access a gadfly server from a remote machine the only
+python modules required (in addition to the standard libraries)
+are gfclient.py and gfsocket.py.
+<p>
+Initialize a connection with a given "POLICY" with "PASSWORD"
+ to a running server
+on "machine.domain.com" using port number 2222 with:
+<pre>
+ from gfclient import gfclient
+ conn = gfclient("POLICY", 2222, "PASSWORD", "machine.domain.com")
+</pre>
+Note that policy names and passwords are case sensitive.
+<p>
+Queries and other statements are normally executed via cursors.
+Obtain a cursor from a connection using:
+<pre>
+ cursor = connection.cursor()
+</pre>
+Execute a statement in a cursor using:
+<pre>
+ cursor.execute(statement)
+</pre>
+or to provide dynamic parameters:
+<pre>
+ cursor.execute(statement, dynamic_parameters)
+</pre>
+For example
+<pre>
+ cursor.execute("select * from work")
+ ...
+ cursor.execute("select * from work where name=?", ("carla",))
+</pre>
+The dynamic parameters work the same as described in the
+<a href="gadfly.html">the main gadfly documentation page</a>.
+In particular INSERT VALUES can insert several rows at once
+by using a list of tuples for the rows.
+<p>
+If there is any problem (bad policy name, bad password, server
+not running, queries not allowed for this policy)
+the execute will generate an exception.
+<p>
+To run a named/prepared query (initialized at startup) use
+execute_prepared, which takes a prepared statement name
+rather than a query string:
+<pre>
+cursor.execute_prepared("updatenan", ("rollingrock", 1))
+...
+cursor.execute_prepared("getnan")
+</pre>
+The execute_prepared method works just like the execute
+method except that the "name" must be the name of a query initialized
+by the startup(...) function at server startup.
+<p><strong>
+NOTE: by default any execution that sends or recieves "too much
+data" will be aborted. Edit gfsocket.py (both on the client end
+and on the server end if different) if you wish to disable this
+sanity check feature.
+<pre>
+LEN_LIMIT=10e8
+</pre></strong>
+<p>
+As with other dbapi cursors the results of a query can be
+extracted as a list of tuples using (after execute):
+<pre>
+ result_list = cursor.fetchall()
+</pre>
+The other fetches (fetchone and fetchmany) have not been
+implemented yet (partially since they don't make much sense
+in this context).
+<p>
+Both named and unnamed statements may be semicolon separated
+sequences of several SQL statements, but if they are they will return
+no results.
+
+<h1>Implementation Comments</h1>
+
+For your information the server/client interaction is much like
+"finger" or "http" -- each client access is a separate TCP/Stream
+connection where the client sends a request and the server sends
+a response. After each access the connection is closed and the
+next access generates a new connection.
+I did it that way, because it was a simple and
+robust strategy (witness the success of HTTP).
+
+<p>
+<strong>
+Please note: Although I have attempted to provide a robust
+implementation
+for this software I do not guarantee its correctness. I hope
+it will work well for you but I do not assume any legal
+responsibility for problems anyone may have during use
+of these programs.
+</strong>
+
+<p>
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+</Body>
+</html>
\ No newline at end of file
=== Zope3/src/zope/app/rdb/gadfly/sql.mar 1.1 => 1.2 ===
<Binary-ish file>
=== Zope3/src/zope/app/rdb/gadfly/sqlbind.py 1.1 => 1.2 === (526/626 lines abridged)
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/sqlbind.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,623 @@
+"""rule bindings for sql grammar."""
+
+def elt0(list, context):
+ """return first member of reduction"""
+ return list[0]
+
+def elt1(list, context):
+ """return second member"""
+ return list[1]
+
+def elt2(list, context):
+ return list[2]
+
+def returnNone(list, context):
+ return None
+
+def stat1(list, context):
+ """return list of len 1 of statements"""
+ return list
+
+#def statn(list, context):
+# """return a list of statement reductions"""
+# [stat, semi, statlist] = list
+# statlist.insert(0, stat)
+# return statlist
+
+def thingcommalist(l, c):
+ [thing, comma, list] = l
+ list.insert(0, thing)
+ return list
+
+def listcommathing(l, c):
+ [list, comma, thing] = l
+ list.append(thing)
+ return list
+
+statn = thingcommalist
+selstat = elt0
+insstat = elt0
+createtablestat = elt0
+droptablestat = elt0
+delstat = elt0
+updatestat = elt0
+createindexstat = elt0
+dropindexstat = elt0
+createviewstat = elt0
+dropviewstat = elt0
[-=- -=- -=- 526 lines omitted -=- -=- -=-]
+ return result
+
+def selectn(list, context):
+ [ selectsubs, comma, select_sublist ] = list
+ (exp, name) = select_sublist
+ selectsubs.addbinding(name, exp)
+ return selectsubs
+
+def selectit(list, context):
+ [exp] = list
+ return (exp, None) # no binding!
+
+def selectname(list, context):
+ [exp, as, alias] = list
+ return (exp, alias)
+
+colalias = elt0
+
+
+#### do the bindings.
+
+# note: all reduction function defs must precede this assign
+VARS = vars()
+
+class punter:
+ def __init__(self, name):
+ self.name = name
+ def __call__(self, list, context):
+ print "punt:", self.name, list
+ return list
+
+class tracer:
+ def __init__(self, name, fn):
+ self.name = name
+ self.fn = fn
+
+ def __call__(self, list, context):
+ print self.name, list
+ return self.fn(list, context)
+
+def BindRules(sqlg):
+ for name in sqlg.RuleNameToIndex.keys():
+ if VARS.has_key(name):
+ #print "binding", name
+ sqlg.Bind(name, VARS[name]) # nondebug
+ #sqlg.Bind(name, tracer(name, VARS[name]) ) # debug
+ else:
+ print "unbound", name
+ sqlg.Bind(name, punter(name))
+ return sqlg
=== Zope3/src/zope/app/rdb/gadfly/sqlgen.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/sqlgen.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,77 @@
+"""grammar generation stuff for sql.
+
+This module does not bind any rule semantics, it
+just generates the parser data structures.
+"""
+
+### interpretation functions and regexen for terminals
+
+MARSHALFILE = "sql.mar"
+
+import string
+alphanum = string.letters+string.digits + "_"
+userdefre = "[%s][%s]*" % (string.letters +"_", alphanum)
+commentre = "--.*"
+
+def userdeffn(str):
+ from string import upper
+ return upper(str)
+
+charstre = "'[^']*'"
+
+def charstfn(str):
+ return str[1:-1]
+
+#numlitre = "[%s][%s\.]*" % (string.digits, alphanum) # not really...
+
+digits = string.digits
+# rely in python to filter out the good/bad/ugly
+intre = "[%s][%s.jJ]*" % (digits,digits)
+numlitre = "%s([Ee][+-]?%s)?" % (intre, intre)
+
+def numlitfn(str):
+ """Note: this is "safe" because regex
+ filters out dangerous things."""
+ return eval(str)
+
+def DeclareTerminals(Grammar):
+ Grammar.Addterm("user_defined_name", userdefre, userdeffn)
+ Grammar.Addterm("character_string_literal", charstre, charstfn)
+ Grammar.Addterm("numeric_literal", numlitre, numlitfn)
+
+def BuildSQL(filename=MARSHALFILE):
+ import kjParseBuild
+ from sqlgram import sqlrules, nonterms, keywords, puncts
+ SQLG = kjParseBuild.NullCGrammar()
+ SQLG.SetCaseSensitivity(0)
+ DeclareTerminals(SQLG)
+ SQLG.Keywords(keywords)
+ SQLG.punct(puncts)
+ SQLG.Nonterms(nonterms)
+ SQLG.comments([commentre])
+ # should add comments
+ SQLG.Declarerules(sqlrules)
+ print "working..."
+ SQLG.Compile()
+ print "testing"
+ from sqlgtest import test
+ for x in test:
+ print SQLG.DoParse1(x)
+ print "dumping to", filename
+ outfile = open(filename, "wb")
+ SQLG.MarshalDump(outfile)
+ outfile.close()
+ return SQLG
+
+def reloadSQLG(filename=MARSHALFILE):
+ """does not bind any interpretation functions."""
+ import kjParser
+ infile = open(filename, "rb")
+ SQLG = kjParser.UnMarshalGram(infile)
+ infile.close()
+ DeclareTerminals(SQLG)
+ return SQLG
+
+def getSQL():
+ from sqlwhere import filename
+ return reloadSQLG(filename)
=== Zope3/src/zope/app/rdb/gadfly/sqlgram.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/sqlgram.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,302 @@
+
+# sql grammar, partial, based on ODBC 2.0 programmer's ref
+
+## someday add subquery precedence to allow more general selects.
+
+sqlrules = """
+
+statement_list ::
+
+@R stat1 :: statement_list >> statement
+@R statn :: statement_list >> statement ; statement_list
+
+@R dropindexstat :: statement >> drop_index_statement
+
+@R createindexstat :: statement >> create_index_statement
+
+@R selstat :: statement >> select_statement
+
+@R insstat :: statement >> insert_statement
+
+@R createtablestat :: statement >> create_table_statement
+
+@R droptablestat :: statement >> drop_table_statement
+
+@R delstat :: statement >> delete_statement_searched
+
+@R updatestat :: statement >> update_statement_searched
+
+@R createviewstat :: statement >> create_view_statement
+
+@R dropviewstat :: statement >> drop_view_statement
+
+## drop view statement
+@R dropview :: drop_view_statement >> DROP VIEW user_defined_name
+
+## create view statement
+@R createview :: create_view_statement >>
+ CREATE VIEW user_defined_name optnamelist AS select_statement
+@R optnamelist0 :: optnamelist >>
+@R optnamelistn :: optnamelist >> ( namelist )
+
+## drop index statement
+@R dropindex :: drop_index_statement >> DROP INDEX user_defined_name
+
+## create index statement
+@R createindex :: create_index_statement >>
+ CREATE INDEX user_defined_name
+ ON user_defined_name
+ ( namelist )
+
+@R createuniqueindex :: create_index_statement >>
+ CREATE UNIQUE INDEX user_defined_name
+ ON user_defined_name
+ ( namelist )
+
+@R names1 :: namelist >> user_defined_name
+@R namesn :: namelist >> namelist , user_defined_name
+
+## update statement
+@R update :: update_statement_searched >>
+ UPDATE user_defined_name
+ SET assns
+ optwhere
+
+@R assn1 :: assns >> assn
+@R assnn :: assns >> assns , assn
+@R assn :: assn >> column_identifier = expression
+
+#####
+
+## delete statement
+@R deletefrom :: delete_statement_searched >> DELETE FROM user_defined_name optwhere
+
+## drop table
+@R droptable :: drop_table_statement >> DROP TABLE user_defined_name
+
+## create table statement ( restricted )
+@R createtable :: create_table_statement >>
+ CREATE TABLE user_defined_name ( colelts )
+@R colelts1 :: colelts >> colelt
+@R coleltsn :: colelts >> colelts , colelt
+@R coleltid :: colelt >> column_definition
+@R coleltconstraint :: colelt >> column_constraint_definition
+## column constraints deferred
+@R coldef :: column_definition >>
+ column_identifier data_type optdefault optcolconstraints
+## optdefault deferred
+@R optdef0 :: optdefault >>
+## optcolconstraint deferred
+@R optcolconstr0 :: optcolconstraints >>
+@R stringtype :: data_type >> character_string_type
+@R exnumtype :: data_type >> exact_numeric_type
+@R appnumtype :: data_type >> approximate_numeric_type
+@R integer :: exact_numeric_type >> INTEGER
+@R float :: approximate_numeric_type >> FLOAT
+@R varchar :: character_string_type >> VARCHAR
+@R varcharn :: character_string_type >> VARCHAR ( numeric_literal )
+
+## insert statement
+
+@R insert1 :: insert_statement >>
+ INSERT INTO table_name optcolids insert_spec
+@R optcolids0 :: optcolids >>
+@R optcolids1 :: optcolids >> ( colids )
+@R colids1 :: colids >> column_identifier
+@R colidsn :: colids >> colids , column_identifier
+@R insert_values :: insert_spec >> VALUES ( litlist )
+@R insert_query :: insert_spec >> sub_query
+@R litlist1 :: litlist >> sliteral
+@R litlistn :: litlist >> litlist , sliteral
+@R sliteral0 :: sliteral >> literal
+@R sliteralp :: sliteral >> + literal
+
+## hack to permit complexes
+
+@R sliterals :: sliteral >> sliteral + literal
+@R sliterald :: sliteral >> sliteral - literal
+@R sliteralm :: sliteral >> - literal
+
+## select statement
+
+@R subselect :: sub_query >>
+ SELECT alldistinct select_list
+ FROM table_reference_list
+ optwhere optgroup opthaving optunion
+
+## @R psubselect :: sub_query >> ( sub_query )
+
+@R selectx :: select_statement >>
+ sub_query
+ optorder_by
+@R ad0 :: alldistinct >>
+@R adall :: alldistinct >> ALL
+@R addistinct :: alldistinct >> DISTINCT
+@R where0 :: optwhere >>
+@R where1 :: optwhere >> WHERE search_condition
+@R group0 :: optgroup >>
+@R group1 :: optgroup >> GROUP BY colnamelist
+@R colnames1 :: colnamelist >> column_name
+@R colnamesn :: colnamelist >> colnamelist , column_name
+@R having0 :: opthaving >>
+@R having1 :: opthaving >> HAVING search_condition
+@R union0 :: optunion >>
+@R union1 :: optunion >> UNION alldistinct sub_query
+@R except1 :: optunion >> EXCEPT sub_query
+@R intersect1 :: optunion >> INTERSECT sub_query
+@R order0 :: optorder_by >>
+@R order1 :: optorder_by >> ORDER BY sortspeclist
+##@R orderby :: order_by_clause >> ORDER BY sortspeclist
+@R sortspec1 :: sortspeclist >> sort_specification
+@R sortspecn :: sortspeclist >> sortspeclist , sort_specification
+## really, should be unsigned int
+@R sortint :: sort_specification >> numeric_literal opt_ord
+@R sortcol :: sort_specification >> column_name opt_ord
+@R optord0 :: opt_ord >>
+@R optordasc :: opt_ord >> ASC
+@R optorddesc :: opt_ord >> DESC
+
+## table reference list (nasty hack alert)
+@R trl1 :: table_reference_list >> user_defined_name
+@R trln :: table_reference_list >> user_defined_name , table_reference_list
+@R trl1a :: table_reference_list >> user_defined_name user_defined_name
+@R trlna :: table_reference_list >> user_defined_name user_defined_name , table_reference_list
+@R trl1as :: table_reference_list >> user_defined_name AS user_defined_name
+@R trlnas :: table_reference_list >> user_defined_name AS user_defined_name , table_reference_list
+
+## select list
+@R selectstar :: select_list >> *
+@R selectsome :: select_list >> selectsubs
+@R select1 :: selectsubs >> select_sublist
+@R selectn :: selectsubs >> selectsubs , select_sublist
+@R selectit :: select_sublist >> expression
+@R selectname :: select_sublist >> expression AS column_alias
+@R colalias :: column_alias >> user_defined_name
+
+## search condition
+@R search1 :: search_condition >> boolean_term
+@R searchn :: search_condition >> boolean_term OR search_condition
+@R bool1 :: boolean_term >> boolean_factor
+@R booln :: boolean_term >> boolean_factor AND boolean_term
+@R bf1 :: boolean_factor >> boolean_primary
+@R notbf :: boolean_factor >> NOT boolean_primary
+@R bp1 :: boolean_primary >> predicate
+@R bps :: boolean_primary >> ( search_condition )
+
+## predicate (simple for now!!!)
+@R predicate1 :: predicate >> comparison_predicate
+
+## comparison predicate (simple for now!!!)
+@R predicateeq :: comparison_predicate >> expression = expression
+@R predicatelt :: comparison_predicate >> expression < expression
+@R predicategt :: comparison_predicate >> expression > expression
+@R predicatele :: comparison_predicate >> expression < = expression
+@R predicatege :: comparison_predicate >> expression > = expression
+@R predicatene :: comparison_predicate >> expression < > expression
+@R predbetween :: comparison_predicate >> expression BETWEEN expression AND expression
+@R prednotbetween :: comparison_predicate >>
+ expression NOT BETWEEN expression AND expression
+
+## exists predicate
+@R predexists :: predicate >> exists_predicate
+@R exists :: exists_predicate >> EXISTS ( sub_query )
+
+## quantified predicate
+@R predqeq :: predicate >> expression = allany ( sub_query )
+@R predqne :: predicate >> expression < > allany ( sub_query )
+@R predqlt :: predicate >> expression < allany ( sub_query )
+@R predqgt :: predicate >> expression > allany ( sub_query )
+@R predqle :: predicate >> expression < = allany ( sub_query )
+@R predqge :: predicate >> expression > = allany ( sub_query )
+@R nnall :: allany >> ALL
+@R nnany :: allany >> ANY
+
+## in predicate
+@R predin :: predicate >> expression IN ( sub_query )
+@R prednotin :: predicate >> expression NOT IN ( sub_query )
+@R predinlits :: predicate >> expression IN ( litlist )
+@R prednotinlits :: predicate >> expression NOT IN ( litlist )
+
+## subquery expression
+@R subqexpr :: expression >> ( sub_query )
+
+## expression (simple for now!!!)
+@R exp1 :: expression >> term
+@R expplus :: expression >> expression + term
+@R expminus :: expression >> expression - term
+@R term1 :: term >> factor
+@R termtimes :: term >> term * factor
+@R termdiv :: term >> term / factor
+@R factor1 :: factor >> primary
+@R plusfactor :: factor >> + factor
+@R minusfactor :: factor >> - factor
+@R primary1 :: primary >> column_name
+@R primarylit :: primary >> literal
+@R primaryexp :: primary >> ( expression )
+@R primaryset :: primary >> set_function_reference
+@R stringlit :: literal >> character_string_literal
+@R stringstring :: literal >> literal character_string_literal
+@R numlit :: literal >> numeric_literal
+
+## set functions (nasty hack!)
+@R countstar :: set_function_reference >> COUNT ( * )
+@R distinctcount :: set_function_reference >> COUNT ( DISTINCT expression )
+@R allcount :: set_function_reference >> COUNT ( expression )
+@R distinctset :: set_function_reference >> aggregate ( DISTINCT expression )
+@R allset :: set_function_reference >> aggregate ( expression )
+@R average :: aggregate >> AVG
+##@R count :: aggregate >> COUNT
+@R maximum :: aggregate >> MAX
+@R minimum :: aggregate >> MIN
+@R summation :: aggregate >> SUM
+@R median :: aggregate >> MEDIAN
+
+## dynamic parameter (varies quite a bit from ODBC spec)
+@R dynamic :: literal >> ?
+
+## column name
+@R columnname1 :: column_name >> column_identifier
+@R columnname2 :: column_name >> table_name . column_identifier
+@R tablename1 :: table_name >> user_defined_name
+@R columnid1 :: column_identifier >> user_defined_name
+"""
+
+nonterms = """
+sliteral
+exists_predicate set_function_reference aggregate
+sortspeclist sort_specification opt_ord
+drop_table_statement delete_statement_searched update_statement_searched
+assns assn
+insert_statement litlist colelt optcolconstraints optdefault
+optcolids insert_spec create_table_statement
+colids colelts column_constraint_definition
+column_definition data_type character_string_type
+exact_numeric_type approximate_numeric_type
+expression term factor primary literal
+comparison_predicate column_alias column_identifier table_name
+boolean_term boolean_factor boolean_primary predicate
+selectsubs expression alias sub_query
+statement_list statement select_statement alldistinct subselect
+select_list table_reference_list optwhere optgroup opthaving
+order_by_clause select_sublist
+optunion optorder_by search_condition colnamelist column_name
+table_reference table_name create_index_statement namelist
+drop_index_statement allany create_view_statement drop_view_statement
+optnamelist
+"""
+
+keywords = """
+INDEX ON ANY IN VIEW AS
+EXCEPT INTERSECT
+EXISTS AVG COUNT MAX MIN SUM MEDIAN
+UPDATE DROP DELETE FROM SET
+INSERT INTO VALUES CREATE TABLE INTEGER FLOAT VARCHAR
+AND OR NOT
+SELECT FROM WHERE HAVING GROUP BY UNION ALL DISTINCT AS ORDER
+ASC DESC BETWEEN UNIQUE
+"""
+
+puncts = """.,*;=<>{}()?+-/"""
+
+# terminals user_defined_name, character_string_literal,
+# numeric_literal
=== Zope3/src/zope/app/rdb/gadfly/sqlgtest.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/sqlgtest.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,17 @@
+"test parses for sql grammar"
+
+test = [
+"select a from x where b=c",
+"select distinct x.a from x where x.b=c",
+"select all a from x where b=c",
+"select a from x, y where b=c or x.d=45",
+"select a as k from x d, y as m where b=c",
+"select 1 as n, a from x where b=c",
+"select * from x",
+"select a from x where b=c",
+"select a from x where not b=c or d=1 and e=5",
+"select a from x where a=1 and (x.b=3 or not b=c)",
+"select -1 from x",
+"select -1e6j from x",
+"insert into table1 (a,b,c) values (-1e6+3j, -34e10, 56j)"
+]
=== Zope3/src/zope/app/rdb/gadfly/sqlmod.py 1.1 => 1.2 === (636/736 lines abridged)
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/sqlmod.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,733 @@
+"""Database modification statement semantics"""
+
+import sqlsem
+
+# ordering of ddef storage is important so, eg, index defs
+# follow table defs.
+
+class Ordered_DDF:
+ """mixin for DDF statement sorting, subclass defines s.cmp(o)"""
+ def __cmp__(self, other):
+ try:
+ #print "comparing", self.name, other.name
+ try:
+ sc = self.__class__
+ oc = other.__class__
+ #print sc, oc
+ except:
+ #print "punting 1", -1
+ return -1
+ if sc in ddf_order and oc in ddf_order:
+ test = cmp(ddf_order.index(sc), ddf_order.index(oc))
+ #print "ddforder", test
+ if test: return test
+ return self.cmp(other)
+ else:
+ test = cmp(sc, oc)
+ #print "punting 2", test
+ return test
+ except:
+ #import sys
+ #print "exception!"
+ #print sys.exc_type, sys.exc_value
+ return -1
+
+ def __coerce__(self, other):
+ return (self, other)
+ def cmp(self, other):
+ """redefine if no name field"""
+ return cmp(self.name, other.name)
+
+CTFMT = """\
+CREATE TABLE %s (
+ %s
+ )"""
+
+class CreateTable(Ordered_DDF):
+ """create table operation"""
[-=- -=- -=- 636 lines omitted -=- -=- -=-]
+ #print "bindings", dynbt.assns
+ return dynbt # ??
+
+class InsertSubSelect(sqlsem.SimpleRecursive):
+
+ def __init__(self, subsel):
+ self.subsel = subsel
+
+ def initargs(self):
+ return (self.subsel,)
+
+ def __repr__(self):
+ return "[subsel] %s" % (self.subsel,)
+
+ def resultexps(self):
+ # get list of result bindings
+ subsel = self.subsel
+ atts = self.subsel.attributes()
+ # bind each as "result.name"
+ exps = []
+ from sqlsem import BoundAttribute
+ for a in atts:
+ exps.append( BoundAttribute("result", a) )
+ return exps # temp
+
+ def relbind(self, db):
+ subsel = self.subsel
+ self.subsel = subsel.relbind(db)
+ # do nothing with domain for now
+ #subsel_domain = subsel.domain()
+ return self
+
+ def eval(self, dyn=None):
+ subsel = self.subsel
+ subsel.uncache()
+ rel = subsel.eval(dyn)
+ tups = rel.rows()
+ from sqlsem import BoundTuple ### temp
+ from sqlsem import kjbuckets
+ kjDict = kjbuckets.kjDict
+ for i in xrange(len(tups)):
+ tupsi = tups[i]
+ new = kjDict()
+ for k in tupsi.keys():
+ new[ ("result", k) ] = tupsi[k]
+ tups[i] = new
+ return tups
+
+# ordering for archiving datadefs
+ddf_order = [CreateTable, CreateIndex, CreateView]
=== Zope3/src/zope/app/rdb/gadfly/sqlsem.py 1.1 => 1.2 === (2852/2952 lines abridged)
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/sqlsem.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,2949 @@
+
+""" sql semantics
+"""
+
+### trim unused methods.
+### make assns use equivalence classes.
+
+### maybe eventually implement disj-conj-eq optimizations
+
+### note: for multithreading x.relbind(...) should ALWAYs return
+### a fresh copy of structure (sometimes in-place now).
+
+### note: binding of order by is dubious with archiving,
+### should not bind IN PLACE, leave unbound elts alone!
+
+### need to fix serialization/deserialization of btand and btor
+###
+
+# use kjbuckets builtin if available
+try:
+ import kjbuckets
+except ImportError:
+ import kjbuckets0
+ kjbuckets = kjbuckets0
+
+Tuple = kjbuckets.kjDict
+Graph = kjbuckets.kjGraph
+Set = kjbuckets.kjSet
+
+import sys, traceback
+### debug
+#sys.stderr = sys.stdin
+
+# operations on simple tuples, mostly from kjbuckets
+#def maketuple(thing):
+# """try to make a tuple from thing.
+# thing should be a dictionary or sequence of (name, value)
+# or other tuple."""
+# from types import DictType
+# if type(thing)==DictType:
+# return Tuple(thing.items() )
+# else: return Tuple(thing)
+
+def no_ints_nulls(list):
+ """in place remove all ints, Nones from a list (for null handling)"""
+ tt = type
+ nn = None
[-=- -=- -=- 2852 lines omitted -=- -=- -=-]
+ kjSet = kjbuckets.kjSet
+ allrows = (kjSet(assns) - kjSet(rows)).items()
+ return allrows
+ op = "EXCEPT"
+
+
+class Parse_Context:
+ """contextual information for parsing
+ p.param() returns a new sequence number for external parameter.
+ """
+ # not serializable
+
+ parameter_index = 0
+
+ # no __init__ yet
+ def param(self):
+ temp = self.parameter_index
+ self.parameter_index = temp+1
+ return temp
+
+ def ndynamic(self):
+ return self.parameter_index
+# update/delete/insert statements
+import sqlmod
+CreateTable = sqlmod.CreateTable
+CreateIndex = sqlmod.CreateIndex
+DropIndex = sqlmod.DropIndex
+DropTable = sqlmod.DropTable
+UpdateOp = sqlmod.UpdateOp
+DeleteOp = sqlmod.DeleteOp
+InsertOp = sqlmod.InsertOp
+InsertValues = sqlmod.InsertValues
+InsertSubSelect = sqlmod.InsertSubSelect
+ColumnDef = sqlmod.ColumnDef
+CreateView = sqlmod.CreateView
+DropView = sqlmod.DropView
+
+# update storage structures from gfdb0
+import gfdb0
+Add_Tuples = gfdb0.Add_Tuples
+Erase_Tuples = gfdb0.Erase_Tuples
+Reset_Tuples = gfdb0.Reset_Tuples
+
+####### testing
+# test helpers
+#def tp(**kw):
+# return maketuple(kw)
+
+#def st(**kw):
+# return BTPredicate(BoundTuple(r=kw))
=== Zope3/src/zope/app/rdb/gadfly/sqlwhere.py 1.1 => 1.2 ===
--- /dev/null Wed Dec 25 09:13:47 2002
+++ Zope3/src/zope/app/rdb/gadfly/sqlwhere.py Wed Dec 25 09:13:13 2002
@@ -0,0 +1,4 @@
+
+'''this module indicates where the sql datastructures are marshalled
+ Auto generated on install: better not touch!
+'''