[Zope-CVS] CVS: Packages/GadflyDA/gadfly - COPYRIGHT:1.1 DLispShort.py:1.1 DumbLispGen.py:1.1 SQLTESTG.py:1.1 __init__.py:1.1 gadfly.html:1.1 gadfly.py:1.1 gfSQL.html:1.1 gfclient.py:1.1 gfdb0.py:1.1 gffaq.html:1.1 gfinstall.py:1.1 gfintrospect.py:1.1 gfrecover.html:1.1 gfserve.py:1.1 gfsocket.py:1.1 gfstest.py:1.1 gftest.py:1.1 idl.py:1.1 index.html:1.1 kjParseBuild.py:1.1 kjParser.py:1.1 kjSet.py:1.1 kjbuckets0.py:1.1 kjpylint.py:1.1 kwParsing.html:1.1 pygram.py:1.1 relalg.py:1.1 remotetest.py:1.1 server.html:1.1 sql.mar:1.1 sqlbind.py:1.1 sqlgen.py:1.1 sqlgram.py:1.1 sqlgtest.py:1.1 sqlmod.py:1.1 sqlsem.py:1.1 sqlwhere.py:1.1
Albertas Agejevas
alga@codeworks.lt
Wed, 7 Aug 2002 06:54:19 -0400
Update of /cvs-repository/Packages/GadflyDA/gadfly
In directory cvs.zope.org:/tmp/cvs-serv15715/gadfly
Added Files:
COPYRIGHT DLispShort.py DumbLispGen.py SQLTESTG.py __init__.py
gadfly.html gadfly.py gfSQL.html gfclient.py gfdb0.py
gffaq.html gfinstall.py gfintrospect.py gfrecover.html
gfserve.py gfsocket.py gfstest.py gftest.py idl.py index.html
kjParseBuild.py kjParser.py kjSet.py kjbuckets0.py kjpylint.py
kwParsing.html pygram.py relalg.py remotetest.py server.html
sql.mar sqlbind.py sqlgen.py sqlgram.py sqlgtest.py sqlmod.py
sqlsem.py sqlwhere.py
Log Message:
Initial implementation of Gadfly DA.
Includes the patched gadfly version from Zope 2 ZGadflyDA
=== Added File Packages/GadflyDA/gadfly/COPYRIGHT ===
The following copyright is modified from the python copyright.
Copyright Notice
----------------
The kjParsing source is copyrighted, but you can freely use and copy it
as long as you don't change or remove the copyright:
Copyright Aaron Robert Watters, 1994
All Rights Reserved
Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee is hereby granted,
provided that the above copyright notice appears in all copies and that
both that copyright notice and this permission notice appear in
supporting documentation.
AARON ROBERT WATTERS DISCLAIMS ALL WARRANTIES WITH REGARD TO
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS, IN NO EVENT SHALL AARON ROBERT WATTERS BE LIABLE
FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
Signature
---------
Aaron Robert Watters
Department of Computer and Information Sciences
New Jersey Institute of Technology
University Heights
Newark, NJ 07102
phone (201)596-2666
fax (201)596-5777
home phone (908)545-3367
email: aaron@vienna.njit.edu
=== Added File Packages/GadflyDA/gadfly/DLispShort.py ===
# Grammar generation
# for lisp lists with strings, ints, vars, print, and setq
# set this variable to regenerate the grammar on each load
REGENERATEONLOAD = 1
import string
GRAMMARSTRING ="""
Value :: ## indicates Value is the root nonterminal for the grammar
@R SetqRule :: Value >> ( setq var Value )
@R ListRule :: Value >> ( ListTail
@R TailFull :: ListTail >> Value ListTail
@R TailEmpty :: ListTail >> )
@R Varrule :: Value >> var
@R Intrule :: Value >> int
@R Strrule :: Value >> str
@R PrintRule :: Value >> ( print Value )
"""
COMPILEDFILENAME = "TESTLispG.py"
MARSHALLEDFILENAME = "TESTLispG.mar"
LISPCOMMENTREGEX = ";.*"
INTREGEX = "["+string.digits+"]+"
STRREGEX = '"[^\n"]*"'
VARREGEX = "["+string.letters+"]["+string.letters+string.digits+"]*"
### declare interpretation functions and regex's for terminals
def intInterp( str ):
return string.atoi(str)
def stripQuotes( str ):
return str[1:len(str)-1]
def echo(string):
return string
def DeclareTerminals(Grammar):
Grammar.Addterm("int", INTREGEX, intInterp)
Grammar.Addterm("str", STRREGEX, stripQuotes)
Grammar.Addterm("var", VARREGEX, echo)
### declare the rule reduction interpretation functions.
def EchoValue( list, Context ):
return list[0]
def VarValue( list, Context ):
varName = list[0]
if Context.has_key(varName):
return Context[varName]
else:
raise NameError, "no such lisp variable in context "+varName
def NilTail( list, Context ):
return []
def AddToList( list, Context ):
return [ list[0] ] + list[1]
def MakeList( list, Context ):
return list[1]
def DoSetq( list, Context):
Context[ list[2] ] = list[3]
return list[3]
def DoPrint( list, Context ):
print list[2]
return list[2]
def BindRules(Grammar):
Grammar.Bind( "Intrule", EchoValue )
Grammar.Bind( "Strrule", EchoValue )
Grammar.Bind( "Varrule", VarValue )
Grammar.Bind( "TailEmpty", NilTail )
Grammar.Bind( "TailFull", AddToList )
Grammar.Bind( "ListRule", MakeList )
Grammar.Bind( "SetqRule", DoSetq )
Grammar.Bind( "PrintRule", DoPrint )
# This function generates the grammar and dumps it to a file.
def GrammarBuild():
import kjParseBuild
LispG = kjParseBuild.NullCGrammar()
LispG.SetCaseSensitivity(0) # grammar is not case sensitive for keywords
DeclareTerminals(LispG)
LispG.Keywords("setq print")
LispG.punct("().")
LispG.Nonterms("Value ListTail")
LispG.comments([LISPCOMMENTREGEX])
LispG.Declarerules(GRAMMARSTRING)
LispG.Compile()
print "dumping as python to "+COMPILEDFILENAME
outfile = open(COMPILEDFILENAME, "w")
LispG.Reconstruct("LispG",outfile,"GRAMMAR")
outfile.close()
print "dumping as binary to "+MARSHALLEDFILENAME
outfile = open(MARSHALLEDFILENAME, "w")
LispG.MarshalDump(outfile)
outfile.close()
BindRules(LispG)
return LispG
# this function initializes the compiled grammar from the generated file.
def LoadLispG():
import TESTLispG
# reload to make sure we get the most recent version!
# (only needed when debugging the grammar).
reload(TESTLispG)
LispG = TESTLispG.GRAMMAR()
DeclareTerminals(LispG)
BindRules(LispG)
return LispG
def unMarshalLispG():
import kjParser
infile = open(MARSHALLEDFILENAME, "r")
LispG = kjParser.UnMarshalGram(infile)
infile.close()
DeclareTerminals(LispG)
BindRules(LispG)
return LispG
########## test the grammar generation
if REGENERATEONLOAD:
print "(re)generating the LispG grammar in file TESTLispG.py"
Dummy = GrammarBuild()
print "(re)generation done."
print "loading grammar as python"
LispG = LoadLispG()
### declare an initial context, and do some tests.
Context = { 'x':3 }
test1 = LispG.DoParse1( '()', Context)
test2 = LispG.DoParse1( '(123)', Context)
test3 = LispG.DoParse1( '(x)', Context)
test4 = LispG.DoParse1( '" a string "', Context)
test5 = LispG.DoParse1( '(setq y (1 2 3) )', Context )
test6 = LispG.DoParse1( '(SeTq x ("a string" "another" 0))', Context )
test7str = """
; this is a lisp comment
(setq abc (("a" x)
("b" (setq d 12))
("c" y) ) ; another lisp comment
)
"""
test7 = LispG.DoParse1( test7str, Context)
test8 = LispG.DoParse1( '(print (1 x d))', Context)
print "unmarshalling the grammar"
LispG2 = unMarshalLispG()
### declare an initial context, and do some tests.
Context = { 'x':3 }
test1 = LispG2.DoParse1( '()', Context)
test2 = LispG2.DoParse1( '(123)', Context)
test3 = LispG2.DoParse1( '(x)', Context)
test4 = LispG2.DoParse1( '" a string "', Context)
test5 = LispG2.DoParse1( '(setq y (1 2 3) )', Context )
test6 = LispG2.DoParse1( '(SeTq x ("a string" "another" 0))', Context )
test7str = """
; this is a lisp comment
(setq abc (("a" x)
("b" (setq d 12))
("c" y) ) ; another lisp comment
)
"""
test7 = LispG2.DoParse1( test7str, Context)
test8 = LispG2.DoParse1( '(print (1 x d))', Context)
=== Added File Packages/GadflyDA/gadfly/DumbLispGen.py ===
#
# test for kjParseBuild module automatic parser generation
#
# lisp lists with strings, ints, vars, and setq
import string
### The string representation for the grammar.
### Since this is used only by GrammarBuild()
### it could be put in a separate file with GrammarBuild()
### to save space/load time after Grammar compilation.
###
GRAMMARSTRING ="""
Value :: ## indicates Value is the root nonterminal for the grammar
@R SetqRule :: Value >> ( setq var Value )
@R ListRule :: Value >> ( ListTail
@R TailFull :: ListTail >> Value ListTail
@R TailEmpty :: ListTail >> )
@R Varrule :: Value >> var
@R Intrule :: Value >> int
@R Strrule :: Value >> str
"""
### the name of the file in which to create the compiled
### grammar declarations
COMPILEDFILENAME = "TESTLispG2.py"
### declare comment form(s) as regular expressions
LISPCOMMENTREGEX = ";.*"
### declare regular expression string constants for terminals
#integer terminal:::::::
INTREGEX = "["+string.digits+"]+"
#string terminal::::::::
STRREGEX = '"[^\n"]*"'
#var terminal::::::::
VARREGEX = "["+string.letters+"]["+string.letters+string.digits+"]*"
### declare interpretation functions for terminals
# int interpretation function: translates string to int:
# Could use string.atoi without the extra level of indirection
# but for demo purposes here it is.
#
def intInterp( str ):
return string.atoi(str)
# interpretation function for strings strips off the surrounding quotes.
def stripQuotes( str ):
if len(str)<2:
TypeError, "string too short?"
return str[1:len(str)-1]
# interpretation function for vars just returns the recognized string
def echo(string):
return string
# This function declares the nonterminals both in the
# "grammar generation phase" and in loading the compiled
# grammar after generation
#
def DeclareTerminals(Grammar):
Grammar.Addterm("int", INTREGEX, intInterp)
Grammar.Addterm("str", STRREGEX, stripQuotes)
Grammar.Addterm("var", VARREGEX, echo)
### declare the rule reduction interpretation functions.
# EchoValue() serves for Intrule and Strrule, since
# we just want to echo the value returned by the
# respective terminal interpretation functions.
#
# Parser delivers list of form [ interpreted_value ]
def EchoValue( list, Context ):
if len(list)!=1:
raise TypeError, "this shouldn't happen! (1)"
return list[0]
# for Varrule interpreter must try to look up the value
# in the Context dictionary
#
# Parser delivers list of form [ var_name ]
def VarValue( list, Context ):
if len(list)!=1:
raise TypeError, "Huh? (2)"
varName = list[0]
if Context.has_key(varName):
return Context[varName]
else:
raise NameError, "no such lisp variable in context "+varName
# for an empty tail, return the empty list
#
# Parser delivers list of form [")"]
def NilTail( list, Context ):
if len(list) != 1 or list[0] != ")":
return TypeError, "Bad reduction?"
return []
# For a full tail, add the new element to the front of the list
#
# Parser delivers list of form [Value, TailValue]
def AddToList( list, Context ):
if len(list) !=2:
return TypeError, "Bad reduction?"
return [ list[0] ] + list[1]
# For a list, simply return the list determined by the tail
#
# Parser delivers list of form ["(", TailValue ]
def MakeList( list, Context ):
if len(list)!=2 or list[0]!="(":
raise TypeError, "Bad reduction? (3)"
return list[1]
# For a setq, declare a new variable in the Context dictionary
#
# Parser delivers list of form # ["(", "setq", varName, Value, ")"]
def DoSetq( list, Context):
if len(list) != 5\
or list[0] != "("\
or list[1] != "setq"\
or list[4] != ")":
print list
raise TypeError, "Bad reduction? (4)"
VarName = list[2]
if type(VarName) != type(''):
raise TypeError, "Bad var name? (5)"
Value = list[3]
# add or set the variable in the Context dictionary
Context[ VarName ] = Value
return Value
# This function Binds the named rules of the Grammar string to their
# interpretation functions in a Grammar.
#
def BindRules(Grammar):
Grammar.Bind( "Intrule", EchoValue )
Grammar.Bind( "Strrule", EchoValue )
Grammar.Bind( "Varrule", VarValue )
Grammar.Bind( "TailEmpty", NilTail )
Grammar.Bind( "TailFull", AddToList )
Grammar.Bind( "ListRule", MakeList )
Grammar.Bind( "SetqRule", DoSetq )
# This function generates the grammar and dumps it to a file.
# Since it will be used only once (after debugging),
# it probably should be put in another file save memory/load-time.
#
# the result returned is a Grammar Object that can be used
# for testing/debugging purposes.
#
# (maybe this should be made into a generic function?)
def GrammarBuild():
import kjParseBuild
# initialize a Null compilable grammar to define
LispG = kjParseBuild.NullCGrammar()
# declare terminals for the grammar
DeclareTerminals(LispG)
# declare the keywords for the grammar
# defun is not used, included here for demo purposes only
LispG.Keywords("setq defun")
# Declare punctuations
# dot is not used here
LispG.punct("().")
# Declare Nonterms
LispG.Nonterms("Value ListTail")
# Declare comment forms
LispG.comments([LISPCOMMENTREGEX])
# Declare rules
LispG.Declarerules(GRAMMARSTRING)
# Compile the grammar
LispG.Compile()
# Write the grammar to a file except for
# the function bindings (which must be rebound)
outfile = open(COMPILEDFILENAME, "w")
LispG.Reconstruct("LispG",outfile,"GRAMMAR")
outfile.close()
# for debugging purposes only, bind the rules
# in the generated grammar
BindRules(LispG)
# return the generated Grammar
return LispG
# this function initializes the compiled grammar from
# generated file.
def LoadLispG():
import TESTLispG2
# make sure we have most recent version (during debugging)
reload(TESTLispG2)
# evaluate the grammar function from generated file
LispG = TESTLispG2.GRAMMAR()
# bind the semantics functions
DeclareTerminals(LispG)
BindRules(LispG)
return LispG
########## test grammar generation
# do generation
Dummy = GrammarBuild()
# load the grammar from the file as LispG
LispG = LoadLispG()
# declare an initial context, and do some tests.
Context = { "x":3 }
test1 = LispG.DoParse1( "()", Context)
test2 = LispG.DoParse1( "(123)", Context)
test3 = LispG.DoParse1( "(x)", Context)
test4 = LispG.DoParse1( '" a string "', Context)
test5 = LispG.DoParse1( "(setq y (1 2 3) )", Context )
test6 = LispG.DoParse1( '(setq x ("a string" "another" 0))', Context )
test7str = """
; this is a lisp comment
(setq abc (("a" x)
("b" (setq d 12))
("c" y) ) ; another lisp comment
)
"""
test7 = LispG.DoParse1( test7str, Context)
=== Added File Packages/GadflyDA/gadfly/SQLTESTG.py ===
# this was used for debugging null productions (a nearly full sql grammar
# is available on request).
#set this to automatically rebuild the grammar.
REBUILD = 1
MARSHALFILE = "SQLTEST.mar"
SELECTRULES = """
## highest level for select statement (not select for update)
select-statement ::
@R selectR :: select-statement >>
SELECT
from-clause
where-clause
group-by-clause
having-clause
## generalized to allow null from clause eg: select 2+2
@R fromNull :: from-clause >>
@R fromFull :: from-clause >> FROM
@R whereNull :: where-clause >>
@R whereFull :: where-clause >> WHERE
@R groupNull :: group-by-clause >>
@R groupFull :: group-by-clause >> GROUP BY
@R havingNull :: having-clause >>
@R havingFull :: having-clause >> HAVING
@R unionNull :: union-clause >>
@R unionFull :: union-clause >> UNION
"""
SELECTNONTERMS = """
select-statement
all-distinct select-list table-reference-list
where-clause group-by-clause having-clause union-clause
maybe-order-by
search-condition column-list maybe-all order-by-clause
column-name from-clause
"""
# of these the following need resolution
# (select-list) (table-reference-list)
# (search-condition) order-by-clause (column-name)
SELECTKEYWORDS = """
SELECT FROM WHERE GROUP BY HAVING UNION DISTINCT ALL AS
"""
# test generation of the grammar
def BuildSQLG():
import kjParseBuild
SQLG = kjParseBuild.NullCGrammar()
SQLG.SetCaseSensitivity(0)
SQLG.Keywords(SELECTKEYWORDS)
SQLG.Nonterms(SELECTNONTERMS)
# no comments yet
SQLG.Declarerules(SELECTRULES)
print "building"
SQLG.Compile()
print "marshaling"
outfile = open( MARSHALFILE, "w")
SQLG.MarshalDump(outfile)
outfile.close()
return SQLG
# load function
def LoadSQLG():
import kjParser
print "unmarshalling"
infile = open(MARSHALFILE, "r")
SQLG = kjParser.UnMarshalGram(infile)
infile.close()
return SQLG
#### for testing
if REBUILD:
SQLG0 = BuildSQLG()
print " rebuilt SQLG0 as compilable grammar"
SQLG = LoadSQLG()
print " build SQLG as reloaded grammar"
=== Added File Packages/GadflyDA/gadfly/__init__.py ===
# Package handling monstrocities
import os
import sqlwhere
# Stuff filename into the namespace so that you don't have to hard code
# this information ahead of time. Should be portable across platforms.
sqlwhere.filename = filename = __path__[0] + os.sep + 'sql.mar'
# Yank the previous namespace in so that nothing breaks
from gadfly import *
=== Added File Packages/GadflyDA/gadfly/gadfly.html === (566/666 lines abridged)
<html>
<head>
<title>
Gadfly: SQL Relational Database in Python, 1.0
</title>
</head>
<body bgcolor="#ffffff">
<table>
<tr>
<td><img src="gadfly.JPG"></td>
<td>
<h1>Gadfly: SQL Relational Database in Python, 1.0</h1>
</td></tr></table>
<blockquote>
Gadfly is a simple relational database system implemented in Python
based on the SQL Structured Query Language.
<p>
<b>
<a href="#install">
The package requires installation before use, please see the section on
installation.</a>
<p>
<a href="gffaq.html">In addition to this general documentation, please
see the Gadfly frequently asked questions</a>
</b>
</blockquote>
</p>
<p>
<ul>
<li>
<a href="#what">What is it?</a>
<li>
<a href="#Why">Why?</a>
<li>
<a href="#Use">Use</a>
<li>
<a href="#Create">Creating a new database</a>
<li>
<a href="#Reconnect">Reconnecting to an existing database</a>
<li>
<a href="#Features">Features</a>
<li>
<a href="#select">The SELECT statement</a>
<li>
<a href="#table">Table creation and "data types"</a>
<li>
<a href="#other">Other supported statements</a>
<li>
<a href="#dyn">Dynamic Values</a>
<li>
[-=- -=- -=- 566 lines omitted -=- -=- -=-]
lola | 6 | lolas
norm | 2 | lolas
woody | 1 | lolas
pierre | 0 | frankies
>>>
</pre>
<h2><a name="arch">Architecture</a></h2>
The SQL grammar is described in sqlgram.py, the binding of the grammar constructs
to semantic objects is performed in sqlbind.py, the semantic objects and their
execution strategies is defined in sqlsem.py. The semantics uses a lot of classical
and non-classical logic (cylindric logic, to be precise) as well as
optimization heuristics
to define a relatively efficient and hopefully correct implementation of SQL.
I recommend the brave have a look at sqlsem.py for a look into the 12 years of
research into databases, logic, and programming languages that contributed bits to
this work. The underlying logic (in a different framework) is given in
<pre>
A. Watters, "Interpreting a Reconstructed Relational Calculus",
ACM SIGMOD Proceedings, 1993, Washington DC, pp. 367-376.
</pre>
The most basic data structures of the implementation are given in either
kjbuckets0.py or the faster
<a href="http://www.chordate.com/kjbuckets/">kjbucketsmodule.c</a>,
which implement the same
data type signatures in Python and in a C extension to Python respectively.
<p>
The gadfly.py module is a simple wrapper that provides a standard DBAPI interface
to the system. The installation script gfinstall.py attempts to install the system,
creating the grammar file sql.mar if needed (or if "forced"). The test suite
gftest.py (which requires a writeable directory argument) attempts to provide
a regression test and a demonstration of the system. The SQL parser also requires
the <a href="http://www.chordate.com/kwParsing">kwParsing</a> parser
generation package, which consists of a number of additional python modules.
<h2><a name="comments">Comments</a></h2>
Please find bugs and report them to <a href="mailto:arw@ifu.net">me</a>.
<p>
The query engine should run faster if you have the builtin
module kjbuckets installed. Otherwise it will use a "python imitation"
kjbuckets0.py. In one test the test suite ran two times faster using kjbuckets.
I suspect it will have a higher payoff for larger data sets.
<p>
<a href="mailto:arw@ifu.net">feedback</a><br>
<a href="../index.html">home</a><br>
<a href="index.html">Gadfly home</a>
</body></html>
=== Added File Packages/GadflyDA/gadfly/gadfly.py ===
"""main entry point for gadfly sql."""
import sqlgen, sqlbind
sql = sqlgen.getSQL()
sql = sqlbind.BindRules(sql)
error = "gadfly_error"
verbosity = 0
class gadfly:
"""as per the DBAPI spec "gadfly" is the connection object."""
closed = 0
verbose = verbosity # debug!
def __init__(self, databasename=None, directory=None,
forscratch=0, autocheckpoint=1, verbose=0):
verbose = self.verbose = self.verbose or verbose
# checkpoint on each commit if set
self.autocheckpoint = autocheckpoint
if verbose:
print "initializing gadfly instance", (\
databasename, directory, forscratch, verbose)
self.is_scratch = forscratch
self.databasename=databasename
self.directory = directory
self.fs = None
self.database = None
# db global transaction id
self.transid = 0
if databasename is not None:
self.open()
def transaction_log(self):
from gfdb0 import Transaction_Logger
if self.verbose:
print "new transaction log for", self.transid
return Transaction_Logger(self.database.log, self.transid, self.is_scratch)
# causes problems in 1.5?
#def __del__(self):
# """clear database on deletion, just in case of circularities"""
# # implicit checkpoint
# if self.verbose:
# print "deleting gadfly instance", self.databasename
# if not self.closed:
# self.close()
def checkpoint(self):
"""permanently record committed updates"""
# note: No transactions should be active at checkpoint for this implementation!
# implicit abort of active transactions!
verbose = self.verbose
if verbose:
print "checkpointing gadfly instance", self.databasename
db = self.database
log = db.log
# dump committed db to fs
fs = self.fs
if fs and db and not db.is_scratch:
# flush the log
if log:
if verbose: print "gadfly: committing log"
log.commit()
elif verbose:
print "gadfly: no log to commit"
if verbose: print "gadfly: dumping mutated db structures"
fs.dump(db)
elif verbose:
print "gadfly: no checkpoint required"
if verbose:
print "gadfly: new transid, reshadowing"
self.transid = self.transid+1
self.working_db.reshadow(db, self.transaction_log())
def startup(self, databasename, directory, scratch=0, verbose=0):
from gfdb0 import Database0, File_Storage0
verbose = self.verbose
if verbose:
print "gadfly: starting up new ", databasename
if self.database:
raise error, "cannot startup, database bound"
self.databasename=databasename
self.directory = directory
db = self.database = Database0()
db.is_scratch = scratch or self.is_scratch
self.fs = File_Storage0(databasename, directory)
self.working_db = Database0(db, self.transaction_log())
# commit initializes database files and log structure
self.commit()
# for now: all transactions serialized
# working db shared among all transactions/cursors
self.transid = self.transid+1
self.working_db = Database0(db, self.transaction_log())
def restart(self):
"""reload and rerun committed updates from log, discard uncommitted"""
# mainly for testing recovery.
if self.verbose:
print "gadfly: restarting database", self.databasename
self.database.clear()
self.working_db.clear()
self.working_db = None
self.database = None
self.open()
def open(self):
"""(re)load existing database"""
if self.verbose:
print "gadfly: loading database", self.databasename
from gfdb0 import File_Storage0, Database0
if self.directory:
directory = self.directory
else:
directory = "."
fs = self.fs = File_Storage0(self.databasename, directory)
db = self.database = fs.load(sql)
self.transid = self.transid+1
self.working_db = Database0(db, self.transaction_log())
def add_remote_view(self, name, definition):
"""add a remote view to self.
Must be redone on each reinitialization!
Must not recursively reenter the query evaluation process for
this database!
"Tables" added in this manner cannot be update via SQL.
"""
self.database[name] = definition
self.working_db[name] = definition
def close(self):
"""checkpoint and clear the database"""
if self.closed: return
if self.verbose:
print "gadfly: closing database", self.databasename
db = self.database
if not db.is_scratch:
self.checkpoint()
if db: db.clear()
wdb = self.working_db
if wdb:
wdb.clear()
self.working_db = None
self.closed = 1
def commit(self):
"""commit the working database+transaction, flush log, new transid"""
verbose = self.verbose
autocheckpoint = self.autocheckpoint
if self.verbose:
print "gadfly: committing", self.transid, self.databasename
self.transid = self.transid+1
fs = self.fs
db = self.database
wdb = self.working_db
wdblog = wdb.log
if wdblog: wdblog.commit()
wdb.commit()
if fs and db and not db.is_scratch:
if autocheckpoint:
if verbose:
print "gadfly: autocheckpoint"
# skips a transid?
self.checkpoint()
else:
if verbose:
print "gadfly: no autocheckpoint"
wdb.reshadow(db, self.transaction_log())
else:
if verbose:
print "gadfly: scratch db, no logging, just reshadow"
wdb.reshadow(db, self.transaction_log())
def rollback(self):
"""discard the working db, new transid, recreate working db"""
verbose = self.verbose
if verbose:
print "gadfly: rolling back", self.transid, self.databasename
if not (self.fs or self.database):
raise error, "unbound, cannot rollback"
# discard updates in working database
self.working_db.clear()
self.transid = self.transid+1
self.working_db.reshadow(self.database, self.transaction_log())
#self.open()
def cursor(self):
if self.verbose:
print "gadfly: new cursor", self.databasename
db = self.database
if db is None:
raise error, "not bound to database"
return GF_Cursor(self)
def dumplog(self):
log = self.database.log
if log:
log.dump()
else:
print "no log to dump"
def table_names(self):
return self.working_db.relations()
def DUMP_ALL(self):
print "DUMPING ALL CONNECTION DATA", self.databasename, self.directory
print
print "***** BASE DATA"
print
print self.database
print
print "***** WORKING DATA"
print
print self.working_db
class GF_Cursor:
verbose = verbosity
arraysize = None
description = None
EVAL_DUMP = 0 # only for extreme debugging!
def __init__(self, gadfly_instance):
verbose = self.verbose = self.verbose or gadfly_instance.verbose
if verbose:
print "GF_Cursor.__init__", id(self)
self.connection = gadfly_instance
self.results = None
self.resultlist = None
self.statement = None
# make a shadow of the shadow db! (in case of errors)
from gfdb0 import Database0
self.shadow_db = Database0()
self.reshadow()
self.connection = gadfly_instance
def reshadow(self):
if self.verbose:
print "GF_Cursor.reshadow", id(self)
db = self.connection.working_db
shadow = self.shadow_db
shadow.reshadow(db, db.log)
if self.verbose:
print "rels", shadow.rels.keys()
def close(self):
if self.verbose:
print "GF_Cursor.close", id(self)
self.connection = None
def reset_results(self):
if self.verbose:
print "GF_Cursor.reset_results", id(self)
rs = self.results
if rs is None:
raise error, "must execute first"
if len(rs)!=1:
raise error, "cannot retrieve multiple results"
rel = rs[0]
rows = rel.rows()
atts = rel.attributes()
tupatts = tuple(atts)
resultlist = list(rows)
if len(tupatts)==1:
att = tupatts[0]
for i in xrange(len(resultlist)):
resultlist[i] = (resultlist[i][att],)
else:
for i in xrange(len(resultlist)):
resultlist[i] = resultlist[i].dump(tupatts)
self.resultlist = resultlist
def fetchone(self):
if self.verbose:
print "GF_Cursor.fetchone", id(self)
r = self.resultlist
if r is None:
self.reset_results()
r = self.resultlist
if len(r)<1:
raise error, "no more results"
result = r[0]
del r[0]
return result
def fetchmany(self, size=None):
if self.verbose:
print "GF_Cursor.fetchmany", id(self)
r = self.resultlist
if r is None:
self.reset_results()
r = self.resultlist
if size is None:
size = len(r)
result = r[:size]
del r[:size]
return result
def fetchall(self):
if self.verbose:
print "GF_Cursor.fetchall", id(self)
return self.fetchmany()
def execute(self, statement=None, params=None):
"""execute operations, commit results if no error"""
success = 0
verbose = self.verbose
if verbose:
print "GF_Cursor.execute", id(self)
if statement is None and self.statement is None:
raise error, "cannot execute, statement not bound"
if statement!=self.statement:
if verbose: print "GF_cursor: new statement: parsing"
# only reparse on new statement.
self.statement=statement
from sqlsem import Parse_Context
context = Parse_Context()
cs = self.commands = sql.DoParse1(statement, context)
else:
if verbose: print "GF_cursor: old statment, not parsing"
cs = self.commands
# always rebind! (db may have changed)
if verbose: print "GF_Cursor: binding to temp db"
# make a new shadow of working db
# (should optimize?)
self.reshadow()
# get shadow of working database
database = self.shadow_db
if self.EVAL_DUMP:
print "***"
print "*** dumping connection parameters before eval"
print "***"
print "*** eval scratch db..."
print
print database
print
print "*** connection data"
print
self.connection.DUMP_ALL()
print "********** end of eval dump"
for i in xrange(len(cs)):
if verbose:
print "GFCursor binding\n", cs[i]
print database.rels.keys()
cs[i] = cs[i].relbind(database)
cs = self.commands
self.results = results = list(cs)
# only unshadow results on no error
try:
for i in xrange(len(cs)):
results[i] = cs[i].eval(params)
success = 1
finally:
#print "in finally", success
# only on no error...
if success:
# commit updates in shadow of working db (not in real db)
if verbose: print "GFCursor: successful eval, storing results in wdb"
database.log.flush()
# database commit does not imply transaction commit.
database.commit()
else:
if verbose:
print \
"GFCursor: UNSUCCESSFUL EVAL, discarding results and log entries"
self.statement = None
self.results = None
self.resultlist = None
database.log.reset()
# handle curs.description
self.description = None
if len(results)==1:
result0 = results[0]
try:
atts = result0.attributes()
except:
pass
else:
descriptions = list(atts)
fluff = (None,) * 6
for i in xrange(len(atts)):
descriptions[i] = (atts[i],) + fluff
self.description = tuple(descriptions)
self.resultlist = None
def setoutputsize(self, *args):
# not implemented
pass
def setinputsizes(self, *args):
# not implemented
pass
def pp(self):
"""return pretty-print string rep of current results"""
from string import join
stuff = map(repr, self.results)
return join(stuff, "\n\n")
=== Added File Packages/GadflyDA/gadfly/gfSQL.html ===
<html>
<head>
<title>
Gadfly SQL constructs
</title>
</head>
<body bgcolor="#ffffff">
<table>
<tr>
<td>
<img src="gadfly.JPG">
</td>
<td>
<h1>Gadfly SQL constructs</h1>
</td></tr></table>
<blockquote>
This document describes SQL constructs supported by Gadfly.
The presentation
does not define the complete syntax -- see sqlgram.py for
the precise syntax as BNF -- nor the complete semantics --
see a good book on SQL for more detailed coverage of semantic
(or use the source, Luke ;c) ).
Also, please have a look at my
<a href="http://mulder.rutgers.edu/~aaron/dbnotes.cgi">evolving
database course notes</a> for more coverage of SQL.
Examples of all supported constructs are also shown in the
test suite source file gftest.py.
This document is only
a very brief guide, primarily of use to those who already
understand something about SQL -- it is neither a tutorial
nor a detailed discussion of syntax and semantics.
</blockquote>
<h1>The Standard, with omissions</h1>
<p>
Gadfly supports a large subset of ODBC 2.0 SQL. The reason
ODBC 2.0 is chosen is because it provides a fairly strong set of
constructs, but does not include some of the more obscure
features of other SQL standards which would be extremely
difficult and complex to implement correctly
(and perhaps, not used very frequently (?)).
<p>
Supported features include views, groupings, aggregates,
subquery expressions, quantified subquery comparisons,
EXISTS, IN, UNION, EXCEPT, INTERSECT, searched mutations and
Indices, among others (see below).
<p>
Some important omissions from ODBC 2.0 at this point are
<pre>
Nulls.
Outer joins.
CHECK conditions.
Enforced data type constraints.
Alter table (can't implement until NULLs arrive).
Date, Time, and Interval data types
</pre>
It is hoped these will be implemented at some future time.
<p>
Less important omissions include
<pre>
Cursor based updates and deletes
(justification: if you really need them the db design
is flawed, and it's possible to use python instead).
LIKE string predicate
(justification: use Python regexes in python code).
Users and permissions
(justification: Gadfly is not intended for full multiuser
use at this time).
</pre>
These may or may not be implemented at some future time.
<h1>Statements</h1>
All interaction with SQL databases is mediated by
SQL statements, or statement sequences. Statement
sequences are statements separated by semicolons.
SQL keywords and user defined names are not case
sensitive (but string values are, of course).
<p>
SQL statements include the following.
<h3>Select Statement</h3>
The select statement derives a table from tables
in the database. It's general form is
<pre>
sub_query
optorder_by
</pre>
Where sub_query is given by
<pre>
SELECT alldistinct select_list
FROM table_reference_list
optwhere
optgroup
opthaving
optunion
</pre>
Read the statement:
<pre>
SELECT [DISTINCT|ALL] expressions or *
FROM tables
[WHERE condition]
[GROUP BY group-expressions]
[HAVING aggregate-condition]
[union-clause]
[ORDER BY columns]
</pre>
as follows:
<pre>
1) Make all combinations of rows from the tables (FROM line)
2) Eliminate those combinations not satisfying condition (WHERE line)
3) (if GROUP present) form aggregate groups that match on group-expressions
4) (if HAVING present) eliminate aggregate groups that don't satisfy
the aggregate-condition.
5) compute the columns to keep (SELECT line).
6) (if union-clause present) combine (union, except, intersect)
the result with the result of another select statement.
7) if DISTINCT, throw out redundant entries.
8) (if ORDER present) order the result by the columns (ascending
or descending as specified, with precedence as listed).
</pre>
This reading has little to do with the actual implementation,
but the answer produced should match this intuitive reading.
<h3>Create and drop table</h3>
The create and drop table constructs
initialize and destroy a table structure, respectively.
<pre>
CREATE TABLE user_defined_name ( colelts )
DROP TABLE user_defined_name
</pre>
The colelts declare the names of the columns for
the table and their data types. The data types are
not checked or enforced in any way at this time.
<h3>Table mutations (INSERT, UPDATE, DELETE)</h3>
Insert, Update, and Delete statements insert rows
into tables, modify rows in tables in place, or
remove rows from tables respectively.
<pre>
INSERT INTO table_name optcolids insert_spec
DELETE FROM user_defined_name optwhere
UPDATE user_defined_name
SET assns
optwhere
</pre>
The insert statement has two variants (in this implementation)
INSERT sub-select and INSERT VALUES.
<pre>
insert into r (a,b,c) select a,b,c from s
insert into r (a,b,c) values (1,2,3)
</pre>
The first inserts
the result of a SELECT statement into the target table
and the other inserts explicit values (which may be dynamic
parameters, see below).
<P>
Cursor based updates are not supported at the SQL level,
eg
<pre>
update r set a=1 where current of curs
</pre>
is not supported.
<h3>Indices</h3>
The create and drop index statements initialize and
destroy index structures respectively.
<pre>
CREATE INDEX user_defined_name
ON user_defined_name
( namelist )
DROP INDEX user_defined_name
</pre>
Indices allow fast access to a table, based on values
for the indexed columns in the namelist.
<p>
Indices can be UNIQUE, meaning that the attributes
of the index cannot take on the same values in the table
twice.
<pre>
CREATE UNIQUE INDEX user_defined_name
ON user_defined_name ( namelist )
</pre>
Unique indices can be used to enforce primary and
secondary key constraints. After a UNIQUE index on
a table is created inserts that attempt to insert
repeat values for the indexed columns will be rejected.
<h3>Views</h3>
Create view and drop view statements initialize and
drop views, respectively.
<pre>
CREATE VIEW user_defined_name optnamelist
AS select_statement
DROP VIEW user_defined_name
</pre>
Views are "derived tables" which are defined
as stored SELECT statements. They can be used
as tables, except that they cannot be directly
mutated.
<p>
It is possible to "implement your own views
in Python". Please see remotetest.py, gfintrospect
and the FAQ for discussion.
<h1>Conditions</h1>
Conditions are truth valued boolean expressions
formed from basic conditions possibly combined using
NOT, AND, OR (where NOT has highest precedence and
OR has lowest precedence) and parentheses.
<p>
Basic conditions include simple comparisons
<pre>
expression = expression
expression < expression
expression > expression
expression <= expression
expression >= expression
expression <> expression
</pre>
Variants of the simple comparisons are the quantified
subquery comparisons
<pre>
expression = ANY ( subquery )
expression = ALL ( subquery )
</pre>
(and similarly for the other comparison operators).
The IN predicate tests membership (like =ANY)
<pre>
expression IN ( subquery )
expression NOT IN ( subquery )
</pre>
For all the quantified comparisons and IN the
subquery must generate a single column table.
<p>
Also included are the the BETWEEN and NOT BETWEEN predicates
<pre>
expression BETWEEN expression AND expression
expression NOT BETWEEN expression AND expression
</pre>
<p>
The most general subquery predicate is EXISTS and NOT EXISTS
which places no restriction on the subquery:
<pre>
EXISTS (subquery)
NOT EXISTS (subquery)
</pre>
<h1>Expressions</h1>
Expressions occur in conditions (WHERE, HAVING, etc.),
in UPDATE searched assignments,
and in the select list of select statements.
<p>
Expressions are formed from primary expressions,
possibly combined using the standard arithmetic operators
and parentheses with the normal precedence.
<p>
Primary expressions include numeric and string literals.
Numeric literals supported are the Python numeric literals.
String constants are set off by apostrophies, where two
apostrophe's in sequence represent an apostrophy in the
string:
<pre>
'SQL string literals ain''t pretty'
</pre>
Column name expressions may be unqualified if they are
unambiguous, or may be qualified with a table name
or table alias
<pre>
bar
frequents.bar
f.bar
</pre>
The rules for scoping of column names are not covered
here. Column names in subqueries may refer to bindings
in the query (or queries) that contain the sub-query.
<p>
Subquery expressions of form
<pre>
( select_statement )
</pre>
must produce a single column and single row table.
<p>
Aggregate operations are only permitted in the select
list or in the HAVING condition of SELECT statements
(including subselects).
<pre>
COUNT(*)
COUNT(expression)
AVG(expression)
MAX(expression)
SUM(expression)
MIN(expression)
</pre>
<em><strong>and also including the non-standard extension MEDIAN
<pre>
MEDIAN(expression)
</pre>
</strong></em>
Aggregate operations can be applied to distinct values
as in
<pre>
COUNT(DISTINCT expression)
</pre>
The Dynamic expression "?" is a placeholder for a value
bound at evaluation time (from Python values). See the
<a href="gadfly.html">
API discussions </a>
(Use) for more details on the use of
dynamic parameters.
<p>
<a href="mailto:arw@ifu.net">feedback</a><br>
<a href="../index.html">home</a><br>
<a href="index.html">Gadfly home</a>
</body>
</html>
=== Added File Packages/GadflyDA/gadfly/gfclient.py ===
"""client access to gadfly server. (gfserve.py)
Imported as a module this module provides interfaces
that remotely access a gadfly database server.
Remote connections: gfclient
connection = gfclient.gfclient(
policy, # the name of the connection policy ["admin" for admin]
port, # the port number the server is running on
password,# the password of the policy
[machine]) # (optional) the machine where server is running
# (defaults to localhost)
methods for gfclient connections:
gfclient.checkpoint() checkpoint the server (fails silently
if connection is not "admin").
gfclient.restart() restart the server (fails silently
if connection is not "admin").
gfclient.shutdown() shutdown the server (fails silently
if connection is not "admin").
cursor = gfclient.cursor() returns a cursor on this connection
methods for cursor objects:
cursor.execute(statement, dynamic_parameters=None)
execute the statement with optional dynamic parameters.
Dynamic parameters can be a list of tuples for INSERT
VALUES statements, otherwise they must be a tuple
of values.
cursor.execute_prepared(name, dynamic_parameters=None)
execute a named statement configured for this connection
policy, with optional dynamic parameters. Dynamic
parameters permitted as for execute depending on the
statement the name refers to.
cursor.fetchall()
return results of the last executed statement
(None for non-queries, or list of tuples).
See gfstest.py for example usage.
SCRIPT INTERPRETATION:
Server maintenance utilities
COMMAND LINE:
python gfclient.py action port admin_password [machine]
TEST EXAMPLE:
python gfclient.py shutdown 2222 admin
action: one of
shutdown: shut down the server with no checkpoint
restart: restart the server (re-read the database and recover)
checkpoint: force a database checkpoint now
port: the port the server is running on
admin_password: the administrative password for the server
machine: [optional] the machine the server runs on.
"""
import gfsocket
def main():
import sys
try:
done=0
argv = sys.argv
[action, port, admin_password] = argv[1:4]
from string import atoi
port = atoi(port)
if len(argv)>4:
machine = argv[4]
else:
machine = None
print action, port, admin_password, machine
if action not in ["shutdown", "restart", "checkpoint"]:
print "bad action", action
print
return
dosimple(action, port, admin_password, machine)
done=1
finally:
if not done:
print __doc__
def dosimple(action, port, pw, machine=None):
import socket
if machine is None:
machine = socket.gethostname()
conn = gfclient("admin", port, pw, machine)
action = getattr(conn, action)
print action()
# copied from gfserve
# shut down the server (admin policy only)
# arguments = ()
# shutdown the server with no checkpoint
SHUTDOWN = "SHUTDOWN"
# restart the server (admin only)
# arguments = ()
# restart the server (recover)
# no checkpoint
RESTART = "RESTART"
# checkpoint the server (admin only)
# arguments = ()
# checkpoint the server
CHECKPOINT = "CHECKPOINT"
# exec prepared statement
# arguments = (prepared_name_string, dyn=None)
# execute the prepared statement with dynamic args.
# autocommit.
EXECUTE_PREPARED = "EXECUTE_PREPARED"
# exec any statement (only if not disabled)
# arguments = (statement_string, dyn=None)
# execute the statement with dynamic args.
# autocommit.
EXECUTE_STATEMENT = "EXECUTE_STATEMENT"
class gfclient:
closed = 0
def __init__(self, policy, port, password, machine=None):
import socket
self.policy = policy
self.port = port
self.password = password
if machine is None:
machine = socket.gethostname()
self.machine = machine
def open_connection(self):
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#print type(sock), sock
sock.connect((self.machine, self.port))
return sock
def send_action(self, action, arguments, socket):
gfsocket.send_certified_action(
self.policy, action, arguments, self.password, socket)
def checkpoint(self):
return self.simple_action(CHECKPOINT)
def simple_action(self, action, args=()):
"""only valid for admin policy: force a server checkpoint"""
sock = self.open_connection()
self.send_action(action, args, sock)
data = gfsocket.recv_data(sock)
data = gfsocket.interpret_response(data)
return data
def restart(self):
"""only valid for admin policy: force a server restart"""
return self.simple_action(RESTART)
def shutdown(self):
"""only valid for admin policy: shut down the server"""
return self.simple_action(SHUTDOWN)
def close(self):
self.closed = 1
def commit(self):
# right now all actions autocommit
pass
# cannot rollback, autocommit on success
rollback = commit
def cursor(self):
"""return a cursor to this policy"""
if self.closed:
raise ValueError, "connection is closed"
return gfClientCursor(self)
class gfClientCursor:
statement = None
results = None
description = None
def __init__(self, connection):
self.connection = connection
# should add fetchone fetchmany
def fetchall(self):
return self.results
def execute(self, statement=None, params=None):
con = self.connection
data = con.simple_action(EXECUTE_STATEMENT, (statement, params))
(self.description, self.results) = data
def execute_prepared(self, name, params=None):
con = self.connection
data = con.simple_action(EXECUTE_PREPARED, (name, params))
if data is None:
self.description = self.results = None
else:
(self.description, self.results) = data
def setoutputsizes(self, *args):
pass # not implemented
def setinputsizes(self):
pass # not implemented
if __name__=="__main__":
main()
=== Added File Packages/GadflyDA/gadfly/gfdb0.py === (1313/1413 lines abridged)
"""storage objects"""
verbosity = 0
import os
# use whatever kjbuckets sqlsem is using
#from sqlsem import kjbuckets, maketuple
# error on checking of data integrity
StorageError = "StorageError"
# use md5 checksum (stub if md5 unavailable?)
def checksum(string):
from md5 import new
return new(string).digest()
def recursive_dump(data, prefix="["):
"""for debugging"""
from types import StringType
if type(data) is StringType:
#print prefix, data
return
p2 = prefix+"["
try:
for x in data:
recursive_dump(x, p2)
except:
print prefix, data
def checksum_dump(data, file):
"""checksum and dump marshallable data to file"""
#print "checksum_dump", file
#recursive_dump(data)
from marshal import dumps, dump
#print "data\n",data
storage = dumps(data)
checkpair = (checksum(storage), storage)
dump(checkpair, file)
def checksum_undump(file):
"""undump marshallable data from file, checksum"""
from marshal import load, loads
checkpair = load(file)
(check, storage) = checkpair
if checksum(storage)!=check:
raise StorageError, "data load checksum fails"
data = loads(storage)
return data
[-=- -=- -=- 1313 lines omitted -=- -=- -=-]
if verbose:
print "recovery successful: clearing log file"
self.clear()
if restart:
if verbose:
print "recreating empty log file"
self.startup()
def read_records(self, file):
"""return log record as (index, (tid, op)) list"""
verbose = self.verbose
if verbose: print "reading log records to error"
import sys
records = {}
from sqlsem import deserialize
count = 0
while 1:
try:
data = checksum_undump(file)
except:
if verbose:
print "record read terminated with error", len(records)
print sys.exc_type, sys.exc_value
break
(transactionid, serial) = data
operation = deserialize(serial)
records[count] = (transactionid, operation)
if verbose:
print count, ": read for", transactionid
print operation
count = count+1
if verbose: print len(records), "records total"
records = records.items()
records.sort()
return records
def dump(self):
verbose = self.verbose
self.shutdown()
print "dumping log"
self.verbose = 1
try:
file = open(self.filename, "rb")
except:
print "DUMP FAILED, cannot open", self.filename
else:
self.read_records(file)
self.verbose = verbose
self.restart()
=== Added File Packages/GadflyDA/gadfly/gffaq.html === (451/551 lines abridged)
<HTML>
<HEAD>
<TITLE>
Chordate Solutions
Gadfly FAQ
</TITLE>
</HEAD>
<BODY>
<TABLE width="700" BORDER="0" CELLSPACING="2" CELLPADDING="6">
<TR>
<TD VALIGN="TOP" rowspan="3">
<IMG SRC="gadfly.JPG" ALT="Chordate" WIDTH="143" HEIGHT="99"></TD>
<TD>
<B>Chordate Systems</B><BR>
Solutions
</TD>
<td valign="TOP" rowspan="2" nowrap>
<a href="#General">General information</a><br>
<a href="#Installation">Installation</a><br>
<a href="#Use">Use</a><br>
<a href="mailto:arw@ifu.net">feedback</a><br>
<a href="../index.html">home</a><br>
<a href="index.html">Gadfly home</a>
</td>
</tr><tr>
<TD>
<em>
Gadfly FAQ (Frequently asked questions)<p>
<a href="#General">General information</a><br>
<UL>
<LI><a href="#what">What is gadfly?</a>
<LI><a href="#version">What is the current version?</a>
<LI><a href="#where">Where can I get it?</a>
<LI><a href="#whysql">Why SQL and the relational model?</a>
<LI><a href="#whypython">Why Python?</a>
<LI><a href="#free">What is the License? Is it Free? Why?</a>
<LI><a href="#bundle">You mean I can bundle it into my own product
for free? Really?</a>
<LI><a href="#platform">Where does it run?</a>
<LI><a href="#req">What are the software/hardware requirements?</a>
<LI><a href="#opt">Is there a query optimizer?</a>
<LI><a href="#fast">Is it fast?</a>
<LI><a href="#limit">Are there data size limitations?</a>
<LI><a href="#recovery">What about recovery after crashes?</a>
<LI><a href="#server">What about client/server based access?</a>
<LI><a href="#concurrency">What about concurrency control?</a>
<LI><a href="#list">Is there a mailing list?</a>
[-=- -=- -=- 451 lines omitted -=- -=- -=-]
<p>
For example gfintrospect.DictKeyValueView will "wrap" a Python dictionary
as a gadfly table and automatically reflect modifications
made by an external Python program to
the dictionary.
<p>
<LI><a name="key">How do you define a primary key?</a>
<p>
New in 1.0 you can get the effect of a primary key by defining
a unique index on the primary key columns
<pre>
create unique index pkey on person(firstname, lastname)
</pre>
Effectively will enforce a primary key constraint for (firstname, lastname)
on the person table.
<p>
<LI><a name="whynot">What about NULLs, Triggers and other missing stuff?</a>
<p>
The present release opted not to add missing standard or non-standard
features that were likely to cause major modifications to large sections
of the implementation, and therefore were likely to introduce bugs.
<p>
<LI><a name="like">Where is the LIKE predicate?</a>
<p>
The LIKE predicate for string matching is still not supported at the
SQL level. For what it's worth, it is easy to use Python's string matching
(regex, re, string.search, etcetera)
facilities on the result of a query. Also, for what it's worth, since
the gadfly optimizer won't easily be able to optimize for string matching
the "by hand" method would essentially be what gadfly would do anyway,
without major modifications to the implementation.
<p>
<LI><a name="crash">After a crash Gadfly won't recover! Help!</a>
<p>
This shouldn't happen, but did happen (at least once) for a previous
release. If it happens again, you can explicitly delete the log files
from the database directory in order to recover the database to a state
which may or may not correspond to the state of the database at the second
to last commit/checkpoint operation. It is possible, but not likely,
that the database state will include some but not all updates from the
last commit, but, to repeat, it shouldn't happen. Please report the
problem if it occurs again.
<p>
</UL>
<a href="mailto:arw@ifu.net">feedback</a><br>
<a href="../index.html">home</a><br>
<a href="index.html">Gadfly home</a>
</BODY>
</HTML>
=== Added File Packages/GadflyDA/gadfly/gfinstall.py ===
#!/usr/local/bin/python
"""Gadfly installation script.
Build the sql grammar.
usage
python <thismodule>
for a simple install or
python <thismodule> force
for a full rebuild (with grammar regeneration).
In the current directory find or create sql.mar and sqlwhere.py
where sql.mar has the marshalled grammar data structures
for parsing sql and sqlwhere.py is a module that indicates
where the grammar file is as value of sqlwhere.filename.
"""
marfile = "sql.mar"
modfile = "sqlwhere.py"
print __doc__
from os import getcwd, path
cwd = getcwd()
modtemplate ="""
'''this module indicates where the sql datastructures are marshalled
Auto generated on install: better not touch!
'''
filename = %s
"""
#wheremod = cwd + "/" + modfile
#where = cwd + "/" + marfile
wheremod = path.join(cwd, modfile)
where = path.join(cwd, marfile)
print
print "now creating", wheremod
f = open(wheremod, "w")
f.write( modtemplate % (`where`,) )
f.close()
from sqlgen import BuildSQL, getSQL
import sys
argv = sys.argv
force = 0
#print argv
if len(argv)>1 and argv[1]=="force":
force = 1
if not force:
try:
sql = getSQL()
except:
print "exception", sys.exc_type, sys.exc_value
print "during load of SQL grammar structures."
print "Apparently the SQL grammar requires regeneration"
force = 1
if force:
print "now generating parser structures (this might take a while)..."
#where = cwd + "/" + marfile
print "building in", where
sql = BuildSQL(where)
print
print "done."
=== Added File Packages/GadflyDA/gadfly/gfintrospect.py ===
"""
View based introspection and extension views
"""
import gfdb0
class RemoteView(gfdb0.View):
"""Virtual superclass. See text for methods and members to define."""
# Usually redefine __init__
def __init__(self):
pass
# set static (static=1) or dynamic (static=0)
# for static tuple seq is generated once per load
# for non-static tuple seq is regenerated once per query
# which uses the view.
static = 0
# define the column_names
column_names = ['Column1']
# define the row generator
def listing(self):
"""return list of values (1 column)
or list of tuples of values (>1 column).
size of elts should match number of columns."""
return [0]
# possibly redefine __repr__ and irepr
def __repr__(self):
return "<Remote View %s at %s>" % (self.__class__, id(self))
irepr = __repr__
# for introspective methods possibly redefine relbind
def relbind(self, db, atts):
return self
### don't touch the following unless you are a guru!
cached_rows = None
def uncache(self):
if self.static: return
self.cached_rows = None
def attributes(self):
from string import upper
return map(upper, self.column_names)
def rows(self, andseqs=0):
cached_rows = self.cached_rows
if cached_rows is None:
tups = list(self.listing())
from sqlsem import kjbuckets
undump = kjbuckets.kjUndump
attributes = tuple(self.attributes())
for i in xrange(len(tups)):
tups[i] = undump(attributes, tups[i])
cached_rows = self.cached_rows = tups
tups = cached_rows[:]
if andseqs:
return (tups, range(len(tups)))
else:
return tups
class DualView(RemoteView):
"""static one row one column view for testing.
(Inspired by Oracle DUAL relation)."""
# trivial example extension view
static = 1
column_names = ['Column1']
def listing(self):
return [0]
class DictKeyValueView(RemoteView):
"""Less trivial example. Dict keys/values converted to strings"""
static = 0 # regenerate in case dict changes
column_names = ["key", "value"]
mapstring = 1
def __init__(self, dict=None):
if dict is None: dict = {}
self.dict = dict
def listing(self):
items = self.dict.items()
if self.mapstring:
def mapper(item):
return tuple(map(str, item))
return map(mapper, items)
else:
return items
class RelationsView(DictKeyValueView):
"""list of relations and whether they are views."""
column_names = ["table_name", "is_view"]
mapstring = 0
def relbind(self, db, atts):
rels = db.rels
dict = {}
for relname in rels.keys():
dict[relname] = rels[relname].is_view
self.dict = dict
return self
class IndicesView(DictKeyValueView):
"""list of indices and relations they index"""
column_names = ["index_name", "table_name", "is_unique"]
mapstring = 0
def relbind(self, db, atts):
rels = db.rels
dict = {}
for relname in rels.keys():
rel = rels[relname]
if not rel.is_view:
index_list = rels[relname].index_list
for index in index_list:
dict[index.name] = (relname, index.unique)
self.dict = dict
return self
def listing(self):
L = []
dict = self.dict
keys = dict.keys()
for k in keys:
L.append( (k,) + dict[k] )
return L
class DataDefsView(DictKeyValueView):
"""Data defs (of non-special views) and definition dumps."""
column_names = ["name", "defn"]
mapstring = 1
def relbind(self, db, atts):
self.dict = db.datadefs
return self
class ColumnsView(RemoteView):
"""table_names and columns therein."""
column_names = ["table_name", "column_name"]
def relbind(self, db, atts):
rels = db.rels
pairs = []
for relname in rels.keys():
for att in rels[relname].attributes():
pairs.append( (relname, att) )
self.pairs = pairs
return self
def listing(self):
return self.pairs
class IndexAttsView(ColumnsView):
"""indices and attributes."""
column_names = ["index_name", "column_name"]
def relbind(self, db, atts):
indices = db.indices
pairs = []
for iname in indices.keys():
for att in indices[iname].attributes():
pairs.append( (iname, att) )
self.pairs = pairs
return self
=== Added File Packages/GadflyDA/gadfly/gfrecover.html ===
<HTML>
<HEAD>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=windows-1252">
<META NAME="Generator" CONTENT="Microsoft Word 97">
<TITLE>Gadfly: recovery</TITLE>
<META NAME="Template" CONTENT="C:\PROGRAM FILES\MICROSOFT OFFICE\OFFICE\html.dot">
</HEAD>
<BODY BGCOLOR="#ffffff">
<table><tr><td>
<img src="gadfly.JPG">
</td><td>
<H1>Gadfly Recovery</H1>
</td></tr></table>
<P>In the event of a software glitch or crash Gadfly may terminate without having stored committed updates.
A recovery strategy attempts to make sure
that the unapplied commited updates are applied when the database restarts.
It is always assumed that there is only one primary (server) process controlling the database (possibly with
multiple clients). </P>
<P>Gadfly uses a simple LOG with deferred updates recovery mechanism. Recovery should be possible in the
presence of non-disk failures (server crash, system crash). Recovery after a disk crash is not available
for Gadfly as yet, sorry. </P>
<P>Due to portability problems Gadfly does not prevent multiple processes from "controlling" the database at
once. For read only access multiple instances are not a problem, but for access with modification, the processes
may collide and corrupt the database. For a read-write database, make sure only one (server) process controls
the database at any given time. </P>
<P>The only concurrency control mechanism that provides serializability for Gadfly as yet is the trivial one --
the server serves all clients serially. This will likely change for some variant of the system at some point. </P>
<P>This section explains the basic recovery mechanism. </P>
<H1>Normal operation</H1>
<H3>Precommit</H3>
<P>During normal operations any active tables are in memory in the process.
Uncommitted updates for a transaction are kept in "shadow tables" until the transaction commits using
<pre>
connection.commit()
</pre>
The shadow tables remember the mutations that have been applied to them. The permanent table copies
are only modified after commit time. A commit commits all updates for all cursors for the connection.
Unless the autocommit feature is disabled (see below) a
commit normally always triggers a checkpoint too.</P>
A rollback
<pre>
connection.rollback()
</pre>
explicitly discards all uncommitted updates and restores the connection to the previously
committed state.</p>
<P>There is a 3rd level of shadowing for statement sequences executed by a cursor.
In particular the design attempts to make sure that if
<pre>
cursor.execute(statement)
</pre>
fails with an error, then the shadow database will contain no updates from
the partially executed statement (which may be a sequence of statements)
but will reflect other completed updates that may have not been committed.
<H3>Commit</H3>
<P>At commit, operations applied to shadow tables are written
out in order of application to a log file before being permanently
applied to the active database. Finally a commit record is written to
the log and the log is flushed. At this point the transaction is considered
committed and recoverable, and a new transaction begins.
Finally the values of the shadow tables replace
the values of the permanent tables in the active database,
(but not in the database disk files until checkpoint, if autocheckpoint
is disabled). </P>
<H3>Checkpoint</H3>
<P>A checkpoint operation brings the persistent copies of the tables on
disk in sync with the in-memory copies in the active database. Checkpoints
occur at server shut down or periodically during server operation.
The checkpoint operation runs in isolation (with no database access
allowed during checkpoint). </P>
<p><em>Note: database connections normally run a checkpoint
after every commit, unless you set
<pre>
connection.autocheckpoint = 0
</pre>
which asks that checkpoints be done explicitly by the program using
<pre>
connection.commit() # if appropriate
connection.checkpoint()
</pre>
Explicit checkpoints should make the database perform better,
since the disk files are written less frequently, but
in order to prevent unneeded (possibly time consuming)
recovery operations after a database
is shutdown and restarted it is important to always execute an explicit
checkpoint at server shutdown, and periodically during long server
runs.</em></p>
<p><strong>Note that if any outstanding operations are uncommitted
at the time of a checkpoint (when autocheckpoint is disabled) the
updates will be lost (ie, it is equivalent to a rollback).
</strong></p>
<P>At checkpoint the old persistent value of each table that has been updated since
the last checkpoint is copied to a back up file, and the currently active value is
written to the permanent table file. Finally if the data definitions have changed
the old definitions are stored to a backup file and the new definitions are written
to the permanent data definition file. To signal successful checkpoint the
log file is then deleted.</P>
<P>
At this point (after log deletion) the database is considered
quiescent (no recovery required). Finally all back up table files are deleted.
[Note, it might be good to keep old logs around... Comments?] </P>
<P>Each table file representation is annotated with a checksum,
so the recovery system can check that the file was stored correctly. </P>
<H1>Recovery</H1>
<P>When a database restarts it automatically determines whether
the last active instance shut down normally and whether recovery
is required. Gadfly discovers the need for recovery by detecting
a non-empty current log file. </P>
<P>To recover the system Gadfly first scans the log file to determine committed transactions.
Then Gadfly rescans the log file applying the operations of committed
transactions to the in memory table values in the order recorded.
When reading in table values for the purpose of recovery Gadfly looks
for a backup file for the table first. If the backup is not corrupt,
its value is used, otherwise the permanent table file is used. </P>
<P>After recovery Gadfly runs a normal checkpoint before resuming
normal operation. </P>
<p>
<strong>
Please note: Although I have attempted to provide a robust
implementation
for this software I do not guarantee its correctness. I hope
it will work well for you but I do not assume any legal
responsibility for problems anyone may have during use
of these programs.
</strong>
<p>
<a href="mailto:arw@ifu.net">feedback</a><br>
<a href="../index.html">home</a><br>
<a href="index.html">Gadfly home</a>
</BODY>
</HTML>
=== Added File Packages/GadflyDA/gadfly/gfserve.py === (429/529 lines abridged)
"""gadfly server mode
script usage
python gfserve.py port database directory password [startup]
test example
python gfserve.py 2222 test dbtest admin gfstest
port is the port to listen to
database is the database to start up. (must exist!)
directory is the directory the database is in.
password is the administrative access password.
startup if present should be the name of a module to use
for startup. The Startup module must contain a function
Dict = startup(admin_policy, connection, Server_instance)
which performs any startup actions on the database needed
and returns either None or a Dictionary of
name > policy objects
where the policy objects describe policies beyond the
admin policy. The startup function may also
modify the admin_policy (disabling queries for example).
The arguments passed to startup are:
admin_policy: the administrative policy
eg you could turn queries off for admin, using admin
only for server maintenance, or you could add prepared
queries to the admin_policy.
connection: the database connection
eg you could perform some inserts before server start
also needed to make policies.
Server_instance
Included for additional customization.
Create policies using
P = gfserve.Policy(name, password, connection, queries=0)
-- for a "secure" policy with only prepared queries allowed,
or
P = gfserve.Policy(name, password, connection, queries=1)
-- for a policy with full access arbitrary statement
execution.
add a "named prepared statement" to a policy using
P[name] = statement
[-=- -=- -=- 429 lines omitted -=- -=- -=-]
print_tb(tb)
self.reply_exception(ServerError,
"unexpected exception: "+exceptiondata, socket)
raise ServerError, exceptiondata
else:
raise ServerError, "unknown action: "+`action`
def certify(self, datastring, certificate, password):
# hook for subclassing
return certify(datastring, certificate, password)
def policy_SHUTDOWN(self, socket):
self.reply_success("attempting server shutdown", socket)
raise SHUTDOWN, "please shut down the server"
def policy_RESTART(self, socket):
self.reply_success("attempting server restart", socket)
raise RESTART, "please restart the server"
def policy_CHECKPOINT(self, socket):
self.reply_success("attempting server checkpoint", socket)
raise CHECKPOINT, "please checkpoint the server"
def policy_EXECUTE_PREPARED(self, name, dyn, socket):
try:
result = self.execute_named(name, dyn)
self.reply_success(result, socket)
except PreparedNameError, detail:
self.reply_exception(PreparedNameError,
"no such prepared statement: "+name,
socket)
def policy_EXECUTE_STATEMENT(self, stat, dyn, socket):
if not self.general_queries:
self.reply_exception(ServerError,
"general statements disallowed on this policy",
socket)
raise ServerError, "illegal statement attempt for: "+self.name
result = self.execute_any_statement(stat, dyn)
self.reply_success(result, socket)
def reply_exception(self, exc, info, socket):
# hook for subclassing
reply_exception(exc, info, socket)
def reply_success(self, data, socket):
# hook for subclassing
reply_success(data, socket)
if __name__=="__main__": main()
=== Added File Packages/GadflyDA/gadfly/gfsocket.py ===
"""socket interactions for gadfly client and server"""
from select import select
# responses
SUCCESS = "SUCCESS"
EXCEPTION = "EXCEPTION"
def reply_exception(exception, info, socket):
"""send an exception back to the client"""
# any error is invisible to client
from gfserve import ServerError
import sys
try:
reply( (EXCEPTION, (exception, info)), socket)
except:
#info = "%s %s" % (sys.exc_type, sys.exc_value)
socket.close()
#raise ServerError, "reply_exception failed: "+`info`
def reply_success(data, socket):
"""report success with data back to client"""
reply( (SUCCESS, data), socket)
def reply(data, socket):
from marshal import dumps
marshaldata = dumps(data)
send_packet(socket, marshaldata)
socket.close()
def send_packet(socket, data):
"""blast out a length marked packet"""
send_len(data, socket)
socket.send(data)
def send_len(data, socket):
"""send length of data as cr terminated int rep"""
info = `len(data)`+"\n"
socket.send(info)
def send_certified_action(actor_name, action, arguments, password, socket):
from marshal import dumps
marshaldata = dumps( (action, arguments) )
cert = certificate(marshaldata, password)
#print actor_name, cert, marshaldata
marshaldata = dumps( (actor_name, cert, marshaldata) )
send_packet(socket, marshaldata)
def unpack_certified_data(data):
from marshal import loads
# sanity check
unpack = (actor_name, certificate, marshaldata) = loads(data)
return unpack
def recv_data(socket, timeout=10):
"""receive data or time out"""
from time import time
endtime = time() + timeout
reader = Packet_Reader(socket)
done = 0
while not done:
timeout = endtime - time()
if timeout<0:
raise IOError, "socket time out (1)"
(readable, dummy, error) = select([socket], [], [socket], timeout)
if error:
raise IOError, "socket in error state"
if not readable:
raise IOError, "socket time out (2)"
reader.poll()
done = (reader.mode==READY)
return reader.data
def interpret_response(data):
"""interpret response data, raise exception if needed"""
from marshal import loads
(indicator, data) = loads(data)
if indicator==SUCCESS:
return data
elif indicator==EXCEPTION:
# ???
raise EXCEPTION, data
else:
raise ValueError, "unknown indicator: "+`indicator`
# packet reader modes
LEN = "LEN"
DATA = "DATA"
READY = "READY"
ERROR = "ERROR"
BLOCK_SIZE = 4028
LEN_LIMIT = BLOCK_SIZE * 10
class Packet_Reader:
"""nonblocking pseudo-packet reader."""
# packets come in as decimal_len\ndata
# (note: cr! not crlf)
# kick too large requests if set
limit_len = LEN_LIMIT
def __init__(self, socket):
self.socket = socket
self.length = None
self.length_remaining = None
self.len_list = []
self.data_list = []
self.received = ""
self.data = None
self.mode = LEN
def __len__(self):
if self.mode is LEN:
raise ValueError, "still reading length"
return self.length
def get_data(self):
if self.mode is not READY:
raise ValueError, "still reading"
return self.data
def poll(self):
mode = self.mode
if mode is READY:
raise ValueError, "data is ready"
if mode is ERROR:
raise ValueError, "socket error previously detected"
socket = self.socket
(readable, dummy, error) = select([socket], [], [socket], 0)
if error:
self.socket.close()
self.mode = ERROR
raise ValueError, "socket is in error state"
if readable:
if mode is LEN:
self.read_len()
# note: do not fall thru automatically
elif mode is DATA:
self.read_data()
def read_len(self):
"""assume socket is readable now, read length"""
socket = self.socket
received = self.received
len_list = self.len_list
if not received:
# 10 bytes at a time until len is read.
received = socket.recv(10)
while received:
# consume, test one char
input = received[0]
received = received[1:]
if input == "\n":
# done reading length
from string import join, atoi
try:
length = self.length = atoi(join(len_list, ""))
except:
self.mode = ERROR
socket.close()
raise ValueError, "bad len string? "+`len_list`
self.received = received
self.length_remaining = length
self.mode = DATA
limit_len = self.limit_len
if limit_len and length>limit_len:
raise ValueError, "Length too big: "+`(length, limit_len)`
return
if len(len_list)>10:
self.mode = ERROR
socket.close()
raise ValueError, "len_list too long: "+`len_list`
len_list.append(input)
if not received:
(readable, dummy, error) = select(\
[socket], [], [socket], 0)
if error:
self.mode = ERROR
socket.close()
raise ValueError, "socket in error state"
if readable:
received = socket.recv(10)
# remember extra data received.
self.received = received
def read_data(self):
# assume socket is readable
socket = self.socket
received = self.received
length_remaining = self.length_remaining
data_list = self.data_list
if received:
data_list.append(received)
self.received = ""
length_remaining = length_remaining - len(received)
recv_len = max(length_remaining, BLOCK_SIZE)
received = socket.recv(recv_len)
if received:
data_list.append(received)
length_remaining = length_remaining - len(received)
if length_remaining<1:
self.mode = READY
from string import join
self.data = join(data_list, "")
self.length_remaining = length_remaining
def certificate(String, password):
"""generate a certificate for a string, using a password"""
from md5 import new
if not String:
raise ValueError, "cannot generate certificate for empty string"
taggedstring = password + String
return new(taggedstring).digest()
def certify(String, cert, password):
"""check a certificate for a string"""
return certificate(String, password) == cert
=== Added File Packages/GadflyDA/gadfly/gfstest.py ===
"""test script for gadfly client and server
Usage: This script interacts with the test database generated
by gftest.py. To start the server from the directory containing
the dbtest directory use:
python gfstest.py start
THIS WILL ONLY WORK IF YOU CREATED THE test DATABASE IN
DIRECTORY dbtest FIRST USING
python gftest.py dbtest
UNLESS YOU RUN THE SERVER IN THE BACKGROUND THE SERVER WILL
HOG THE WINDOW YOU STARTED IT IN AND YOU WILL HAVE TO USE ANOTHER
WINDOW UNTIL THE SERVER IS SHUT DOWN (SEE BELOW).
Then from *anywhere* (on the same machine) access the database
using
python gfstest.py restart
- restart the server (reread the database)
python gfstest.py checkpoint
- force checkpoint the server
python gfstest.py queries
- run some example queries and updates
python gfstest.py policy_test
- test the policies test and test1 created by the startup
function in this module.
python gfstest.py bogusshutdown
- attempt to shut down the server with a bogus password
[should generate an exception]
...and finally
python gfstest.py shutdown
- shut down the server for real.
As mentioned the startup function of this module illustrates
how to create a "startup" function for a server and initialize policy
objects with named, prepared queries.
"""
PORT = 2222
DB = "test"
DBDIR = "dbtest"
PW = "admin"
STARTUP = "gfstest"
import sys, socket
def main():
argv = sys.argv
command = argv[1]
if command=="start":
print "attempting to start the server"
from gfserve import Server
print "making a server on", PORT, DB, DBDIR, PW, STARTUP
S = Server(PORT, DB, DBDIR, PW, STARTUP)
print "initializing the server"
S.init()
print "starting the server", S.connection
S.start()
elif command=="shutdown":
dosimple("shutdown", PW)
elif command=="bogusshutdown":
print "BOGUS shutdown attempt"
dosimple("shutdown", "bad password")
elif command=="restart":
dosimple("restart", PW)
elif command=="checkpoint":
dosimple("checkpoint", PW)
elif command=="queries":
doqueries()
elif command=="policy_test":
policy_test()
else:
print "unknown command", command
print __doc__
def policy_test():
"""test the test1 and test policies"""
print "testing non-admin policies test and test1"
from gfclient import gfclient
import sys
machine = socket.gethostname()
conn = gfclient("test", PORT, "test", machine)
cursor = conn.cursor()
print "testing test policy: nan values before:"
cursor.execute_prepared("getnan")
for x in cursor.fetchall():
print x
print "updating nan"
cursor.execute_prepared("updatenan", ("pabst", 4))
print "nan after"
cursor.execute_prepared("getnan")
for x in cursor.fetchall():
print x
print "updating nan again"
cursor.execute_prepared("updatenan", ("rollingrock", 1))
print "trying an illegal update"
try:
cursor.execute("delete from frequents")
except:
print "exception", sys.exc_type, sys.exc_value
print "as expected"
else:
raise "DAMN!", "illegal query apparently completed!!!"
print; print "testing policy test1"; print
conn = gfclient("test1", PORT, "test1", machine)
cursor = conn.cursor()
print "getting norm"
cursor.execute_prepared("qlike", ("norm",))
print cursor.description
for x in cursor.fetchall():
print x
print "trying an illegal query again"
try:
cursor.execute("create table test(name varchar)")
except:
print "exception", sys.exc_type, sys.exc_value
print "as expected"
else:
raise "Damn!(2)", "illegal query apparently completed"
def startup(admin_policy, connection, Server_instance):
"""example startup script.
add a policies test and test1 passwords same
test1 is allowed to query the frequents table by name
test is allowed to update likes where drinker='nan'
also add prepared query dumpwork to admin_policy.
"""
from gfserve import Policy
admin_policy["dumpwork"] = "select * from work"
test1 = Policy("test1", "test1", connection, queries=0)
test = Policy("test", "test", connection, queries=0)
test1["qlike"] = "select * from likes where drinker=?"
test["updatenan"] = """
update likes
set beer=?, perday=?
where drinker='nan'
"""
test["getnan"] = """
select * from likes where drinker='nan'
"""
return {"test": test, "test1": test1}
def doqueries():
print "executing queries and updates"
from gfclient import gfclient
import sys
machine = socket.gethostname()
conn = gfclient("admin", PORT, PW, machine)
cursor = conn.cursor()
for q in admin_queries:
print;print;print q;print
try:
cursor.execute(q)
except:
print "exception in execute"
print sys.exc_type
v = sys.exc_value
from types import TupleType, ListType
if type(v) in (TupleType, ListType):
for x in v: print x
else:
print v
else:
print "executed"
#print q
print "description"
print cursor.description
print "results"
try:
r = cursor.fetchall()
if r is None:
print "no results"
else:
for x in r:
print x
except:
print "exception in results"
print sys.exc_type, sys.exc_value
print dir(cursor)
# try dumpwork
print; print; print "dumpwork"; print
cursor.execute_prepared("dumpwork")
for x in cursor.fetchall():
print x
# try dynamic parameters
stat = """
select distinct drinker
from likes l, serves s
where l.beer = s.beer and s.bar=?
"""
print; print stat; print "dynamic query ?=cheers"
cursor.execute(stat, ("cheers",))
for x in cursor.fetchall():
print x
admin_queries = [
"""select count(*) from work""",
"""select * from frequents""",
"""select count(*) from frequents""",
"""select count(drinker) from frequents""",
"""insert into frequents(drinker, bar, perweek)
values ('sally', 'cheers', 2)""",
"""select * from frequents""",
"""select syntax error from work""",
"""select drinker, count(bar) from frequents
group by drinker""",
]
def dosimple(command, pw):
print "attempting remote %s (%s) for server on local machine" % (command, pw)
from gfclient import gfclient
machine = socket.gethostname()
conn = gfclient("admin", PORT, pw, machine)
action = getattr(conn, command)
print action()
if __name__=="__main__":
main()
=== Added File Packages/GadflyDA/gadfly/gftest.py === (624/724 lines abridged)
"""test script for gadfly
usage gftest.py <directory>
run in current directory creates a database in files
test.dfs LIKES.grl SERVES.grl FREQUENTS.grl
"""
def test(directory):
print "testing"
from gadfly import gadfly
connect = gadfly()
connect.startup("test", directory)
curs = connect.cursor()
print
print "TABLE CREATES"
for x in table_creates:
print x
curs.execute(x)
curs.execute("create table empty (nothing varchar)")
C = """
CREATE TABLE work (
name VARCHAR,
hours INTEGER,
rate FLOAT)
"""
print C
curs.execute(C)
print
C = """
CREATE TABLE accesses (
page VARCHAR,
hits INTEGER,
month INTEGER)
"""
print C
curs.execute(C)
print
print "INSERTS"
C = """
INSERT INTO work(name, hours, rate) VALUES (?, ?, ?)
"""
D = [
("sam", 30, 40.2),
("norm", 45, 10.2),
("woody", 80, 5.4),
("diane", 3, 4.4),
("rebecca", 120, 12.9),
("cliff", 26, 200.00),
("carla", 9, 3.5),
[-=- -=- -=- 624 lines omitted -=- -=- -=-]
postresults.append(cursor.fetchall())
print cursor.pp()
except:
d = sys.exc_type
print "*** exception", d
postresults.append(d)
if preresults==postresults:
print "*** same results as before uncommitted updates"
else:
print "*** differing results from before uncommitted updates"
if dummy==1:
print "*** RESTART: DUMPLOG"
connect.dumplog()
print "*** RESTARTING (RECOVER FROM LOG, DISCARD UNCOMMITTED)"
connect.restart()
def retest(directory):
print "*" * 30
print "*** reconnect test"
from gadfly import gadfly
connect = gadfly("test", directory)
cursor = connect.cursor()
for s in updates:
print; print
print s
if s in trace_updates:
cursor.EVAL_DUMP = 1
cursor.execute(s)
cursor.EVAL_DUMP = 0
print cursor.pp()
#print; print "CONNECTION DATA BEFORE COMMIT"
#connect.DUMP_ALL()
connect.commit()
#print; print "CONNECTION DATA AFTER COMMIT"
#connect.DUMP_ALL()
connect.close()
return connect
if __name__=="__main__":
import sys
argv = sys.argv
if len(argv)<2:
print "USAGE: python <thismodule> <db_directory>"
print " please provide a directory for test database!"
else:
directory = argv[1]
test(directory)
rollbacktest(directory)
retest(directory)
=== Added File Packages/GadflyDA/gadfly/idl.py ===
# idl grammar
#
# Note, this grammar requires a special hack at the lexical
# level in order to parse the fragment
#
# ...
# case abc::def: jjj::www: whatever...
#
# (Yuck!)
# Some would argue this is a language design flaw, but whatever...
# It causes a shift/reduce problem without special handling for ::
# below coloncolon is a 'fake' keyword that parses as two colons.
idlgramstring = """
specification ::
## 1
@R r1a :: specification >> definition speclist
@R r1b :: speclist >> specification
@R r1c :: speclist >>
## 2 punct ;
@R r2a :: definition >> type_dcl ;
@R r2b :: definition >> const_dcl ;
@R r2c :: definition >> except_dcl ;
@R r2d :: definition >> interface_nt ;
@R r2e :: definition >> module_nt ;
## 3 identifier=term, module=kw puncts {}
@R r3 :: module_nt >> module identifier { specification }
## 4
@R r4a :: interface_nt >> interface_dcl
@R r4b :: interface_nt >> forward_dcl
## 5
@R r5 :: interface_dcl >> interface_header { interface_body }
## 6 interface=kw
@R r6 :: forward_dcl >> interface identifier
## 7 puncts []
@R r7 :: interface_header >> interface identifier [ inheritance_spec ]
## 8
@R r8a :: interface_body >>
@R r8b :: interface_body >> export interface_body
## 9
@R r9a :: export >> type_dcl
@R r9b :: export >> const_dcl
@R r9c :: export >> except_dcl
@R r9d :: export >> attr_dcl
@R r9e :: export >> op_dcl
## 10 punct ,:
@R r10a :: inheritance_spec >> : scoped_name_list
@R r10b :: scoped_name_list >> scoped_name
@R r10c :: scoped_name_list >> scoped_name_list , scoped_name
## 11
@R r11a :: scoped_name >> identifier
@R r11b :: scoped_name >> colon_colon identifier
@R r11d :: scoped_name >> scoped_name coloncolon identifier
## 12 const=kw punct =
@R r12 :: const_dcl >> const const_type identifier = const_expr
## 13
@R r13a :: const_type >> integer_type
@R r13b :: const_type >> char_type
@R r13c :: const_type >> boolean_type
@R r13d :: const_type >> floating_type
@R r13e :: const_type >> string_type
@R r13f :: const_type >> scoped_name
## 14
@R r14 :: const_expr >> or_expr
##15 punct |
@R r15a :: or_expr >> xor_expr
@R r15b :: or_expr >> or_expr | xor_expr
##16 punct ^
@R r16a :: xor_expr >> and_expr
@R r16b :: xor_expr >> xor_expr ^ and_expr
##17 punct &
@R r17a :: and_expr >> shift_expr
@R r17b :: and_expr >> and_expr & shift_expr
##18 punct > <
@R r18a :: shift_expr >> add_expr
@R r18b :: shift_expr >> shift_expr > > add_expr
@R r18c :: shift_expr >> shift_expr < < add_expr
##19 punct +-
@R r19a :: add_expr >> mult_expr
@R r19b :: add_expr >> add_expr + mult_expr
@R r19c :: add_expr >> add_expr - mult_expr
##20 punct */%
@R r20a :: mult_expr >> unary_expr
@R r20b :: mult_expr >> mult_expr * unary_expr
@R r20c :: mult_expr >> mult_expr / unary_expr
@R r20d :: mult_expr >> mult_expr % unary_expr
##21
@R r21a :: unary_expr >> unary_operator primary_expr
@R r21b :: unary_expr >> primary_expr
##22
@R r22a :: unary_operator >> -
@R r22b :: unary_operator >> +
@R r22c :: unary_operator >> ~
##23 punct ()
@R r23a :: primary_expr >> scoped_name
@R r23b :: primary_expr >> literal
@R r23c :: primary_expr >> ( const_expr )
##24 terms = *_literal (?) except boolean
@R r24a :: literal >> integer_literal
@R r24b :: literal >> string_literal
@R r24c :: literal >> character_literal
@R r24d :: literal >> floating_pt_literal
@R r24e :: literal >> boolean_literal
##25 kw TRUE FALSE
@R r25a :: boolean_literal >> TRUE
@R r25b :: boolean_literal >> FALSE
##26
@R r26 :: positive_int_literal >> const_expr
##27 kw typedef
@R r27a :: type_dcl >> typedef type_declarator
@R r27b :: type_dcl >> struct_type
@R r27c :: type_dcl >> union_type
@R r27d :: type_dcl >> enum_type
##28
@R r28 :: type_declarator >> type_spec declarators
##29
@R r29a :: type_spec >> simple_type_spec
@R r29b :: type_spec >> constr_type_spec
##30
@R r30a :: simple_type_spec >> base_type_spec
@R r30b :: simple_type_spec >> template_type_spec
@R r30c :: simple_type_spec >> scoped_name
##31
@R r31a :: base_type_spec >> floating_pt_type
@R r31b :: base_type_spec >> integer_type
@R r31c :: base_type_spec >> char_type
@R r31d :: base_type_spec >> boolean_type
@R r31e :: base_type_spec >> octet_type
@R r31f :: base_type_spec >> any_type
## 32
@R r32a :: template_type_spec >> sequence_type
@R r32b :: template_type_spec >> string_type
##33
@R r33a :: constr_type_spec >> struct_type
@R r33b :: constr_type_spec >> union_type
@R r33c :: constr_type_spec >> enum_type
##34
@R r34a :: declarators >> declarator
@R r34b :: declarators >> declarators , declarator
##35
@R r35a :: declarator >> simple_declarator
@R r35b :: declarator >> complex_declarator
##36
@R r36 :: simple_declarator >> identifier
##37
@R r37 :: complex_declarator >> array_declarator
##38 kw float double
@R r38a :: floating_pt_type >> float
@R r38b :: floating_pt_type >> double
##39
@R r39a :: integer_type >> signed_int
@R r39b :: integer_type >> unsigned_int
##40
@R r40 :: signed_int >> signed_long_int
@R r40 :: signed_int >> signed_short_int
##41 kw long
@R r41 :: signed_long_int >> long
##42 kw short
@R r42 :: signed_short_int >> short
##43
@R r43 :: unsigned_int >> unsigned_long_int
@R r43 :: unsigned_int >> unsigned_short_int
##44 kw unsigned
@R r44 :: unsigned_long_int >> unsigned long
##45
@R r45 :: unsigned_short_int >> unsigned short
##46 kw char
@R r46 :: char_type >> char
##47 kw boolean
@R r47 :: boolean_type >> boolean
##48 kw octet
@R r48 :: octet_type >> octet
##49 kw any
@R r49 :: any_type >> any
##50 kw struct
@R r50 :: struct_type >> struct identifier { member_list }
##51
@R r51a :: member_list >> member
@R r51b :: member_list >> member_list member
##52
@R r52 :: member >> type_spec declarators ;
##53 kw union switch
@R r53 :: union_type >>
union identifier switch ( switch_type_spec ) { switch_body }
##54
@R r54a :: switch_type_spec >> integer_type
@R r54b :: switch_type_spec >> char_type
@R r54c :: switch_type_spec >> boolean_type
@R r54d :: switch_type_spec >> enum_type
@R r54e :: switch_type_spec >> scoped_name
##55
@R r55a :: switch_body >> case_nt
@R r55b :: switch_body >> switch_body case_nt
##56
@R r56a :: case_nt >> case_labels element_spec ;
@R r56b :: case_labels >> case_label
@R r56c :: case_labels >> case_labels case_label
##57 kw default case
@R r57a :: case_label >> case const_expr :
@R r57b :: case_label >> default :
##58
@R r58 :: element_spec >> type_spec declarator
##59 kw enum
@R r59a :: enum_type >> enum identifier { enumerators }
@R r59b :: enumerators >> enumerator
@R r59c :: enumerators >> enumerators , enumerator
##60
@R r60 :: enumerator >> identifier
##61 kw sequence
@R r61 :: sequence_type >> sequence < simple_type_spec , positive_int_const >
##62 kw string
@R r62a :: string_type >> string < positive_int_const >
@R r62b :: string_type >> string
##63
@R r63a :: array_declarator >> identifier fixed_array_sizes
@R r63b :: fixed_array_sizes >> fixed_array_size
@R r63c :: fixed_array_sizes >> fixed_array_sizes fixed_array_size
##64
@R r64 :: fixed_array_size >> [ positive_int_const ]
##65 kw attribute readonly
@R r65a :: attr_dcl >> maybe_readonly attribute param_type_spec simple_declarators
@R r65b :: maybe_readonly >> readonly
@R r65c :: maybe_readonly >>
@R r65d :: simple_declarators >> simple_declarator
@R r65e :: simple_declarators >> simple_declarators , simple_declarator
##66 kw exception
@R r66a :: except_dcl >> exception identifier { members }
@R r66b :: members >>
@R r66c :: members >> member_list
##67
@R r67a :: op_dcl >>
maybe_op_attribute op_type_spec identifier parameter_dcls
maybe_raises_expr maybe_context_expr
@R r67b :: maybe_op_attribute >>
@R r67c :: maybe_op_attribute >> op_attribute
@R r67d :: maybe_raises_expr >>
@R r67e :: maybe_raises_expr >> raises_expr
@R r67f :: maybe_context_expr >>
@R r67g :: maybe_context_expr >> context_expr
##68 kw oneway
@R r68a :: op_attribute >> oneway
##69 kw void
@R r69a :: op_type_spec >> param_type_spec
@R r69b :: op_type_spec >> void
##70
@R r70a :: parameter_dcls >> ( parameterlist )
@R r70b :: parameter_dcls >> ( )
@R r70c :: parameterlist >> param_dcl
@R r70d :: parameterlist >> parameterlist , param_dcl
##71
@R r71 :: param_dcl >> param_attribute param_type_spec simple_declarator
##72 kw in out inout
@R r72 :: param_attribute >> in
@R r72 :: param_attribute >> out
@R r72 :: param_attribute >> inout
##73 kw raises
@R r73 :: raises_expr >> raises ( scoped_name_list )
##74 kw context
@R r74 :: context_expr >> context ( string_literal_list )
@R r74b :: string_literal_list >> string_literal
@R r74c :: string_literal_list >> string_literal_list , string_literal
@R r75 :: param_type_spec >> base_type_spec
@R r75 :: param_type_spec >> string_type
@R r75 :: param_type_spec >> scoped_name
"""
nonterms = """
colon_colon
param_attribute
unsigned_long_int unsigned_short_int param_dcl
parameterlist string_literal_list
members maybe_op_attribute maybe_raises_expr maybe_context_expr
op_type_spec parameter_dcls op_attribute raises_expr context_expr
maybe_readonly param_type_spec simple_declarators simple_declarator
fixed_array_sizes fixed_array_size
element_spec enumerator enumerators
switch_type_spec switch_body case_nt case_labels case_label
member_list member
signed_int unsigned_int signed_long_int signed_short_int
simple_declarator complex_declarator array_declarator
declarator
sequence_type string_type
floating_pt_type integer_type char_type boolean_type
octet_type any_type
base_type_spec template_type_spec
simple_type_spec constr_type_spec
type_spec declarators
type_declarator struct_type union_type enum_type
literal boolean_literal positive_int_literal
mult_expr unary_expr unary_operator primary_expr
or_expr xor_expr and_expr shift_expr add_expr
integer_type char_type boolean_type floating_type string_type
const_type const_expr
scoped_name_list scoped_name
attr_dcl op_dcl
inheritance_spec export
interface_header interface_body
interface_dcl forward_dcl
type_dcl const_dcl except_dcl interface_nt module_nt
specification definition speclist
"""
keywords = """
exception oneway void in out inout raises context
interface module const TRUE FALSE typedef float double long
unsigned short char boolean octet any struct union switch
enum string attribute readonly default case sequence ::
"""
# NOTE: FOR NECESSARY HACKERY REASONS :: IS A KEYWORD!
punctuations = ";{}()[],:|^&<>+-*/%~="
# dummy regexen
identifierre = "identifier"
integer_literalre = "123"
positive_int_constre = "999"
string_literalre = "'string'"
character_literalre= "'c'"
floating_pt_literalre = "1.23"
# dummy interp fun for all terminals
def echo (str):
return str
def DeclareTerminals(Grammar):
Grammar.Addterm("identifier", identifierre, echo)
Grammar.Addterm("integer_literal", integer_literalre, echo)
Grammar.Addterm("string_literal", string_literalre, echo)
Grammar.Addterm("character_literal", character_literalre, echo)
Grammar.Addterm("floating_pt_literal", floating_pt_literalre, echo)
Grammar.Addterm("positive_int_const", positive_int_constre, echo)
## we need to override LexDictionary to recognize :: as a SINGLE punctuation.
## (not possible using standard kjParsing, requires a special override)
import kjParser
class myLexDictionary(kjParser.LexDictionary):
def __init__(self):
kjParser.LexDictionary.__init__(self)
map = ((kjParser.KEYFLAG, "coloncolon"), "coloncolon")
self.keywordmap["::"] = map
self.keywordmap["coloncolon"] = map
def Token(self, String, StartPosition):
if String[StartPosition:StartPosition+2] == "::":
tok = self.keywordmap["::"]
return (tok, 2)
# default:
return kjParseBuild.LexDictionary.Token(self, String, StartPosition)
# default bind all rules
def GrammarBuild():
import kjParseBuild
idl = kjParseBuild.NullCGrammar()
idl.LexD = myLexDictionary()
#idl.SetCaseSensitivity(0) # grammar is not case sensitive for keywords
DeclareTerminals(idl)
idl.Keywords(keywords)
idl.punct(punctuations)
idl.Nonterms(nonterms)
#idl.comments([LISPCOMMENTREGEX])
idl.Declarerules(idlgramstring)
print "now compiling"
idl.Compile()
return idl
if __name__=="__main__": GrammarBuild()
=== Added File Packages/GadflyDA/gadfly/index.html ===
<HTML>
<HEAD>
<TITLE>
Gadfly/kwParsing Downloads and Documentation
</TITLE>
</HEAD>
<BODY bgcolor="#ffffff">
<table>
<tr>
<td valign="top"><img src="gadfly.JPG"></td>
<td>
<h1>Gadfly 1.0: An SQL Database in Python</h1>
Gadfly/kwParsing downloads and documentation.
<p>
<b>Gadfly requires the kwParsing parser/parser generator package.
The "two packages" are released as one distribution (kwP) with the
same generous copyright.
<p>
<a href="kwP.tar.gz">Download kwP.tar.gz in tar-gz format with tar.gz extension.</a>
<p>
<a href="kwP.tgz">Download kwP.tgz in tar-gz format with tgz extension.</a>
<p>
<a href="kjbuckets.pyd">Separately download kjbuckets.pyd data structure
accellerator extension module for Python 1.5 on Windows NT or Windows 95/98
only.</a>
</b>
<p>
[The tar.gz extension doesn't always work for MSIE browsers.]
<p>
<a href="gffaq.html">Gadfly frequently asked questions (FAQ).</a>
<p>
<a href="gadfly.html">General information with links to other documentation,
<em>including installation instructions</em>.</a>
</td>
</tr></table>
<hr><br>
<table BORDER>
<tr>
<td width="40%">
<a href="gadfly.py">gadfly.py</a> main module<br>
<a href="gfserve.py">gfserve.py</a> main TCP/IP server mode<br>
<a href="gfclient.py">gfclient.py</a> main TCP/IP client mode<br>
<a href="gfsocket.py">gfsocket.py</a> client/server support module<br>
<a href="gfstest.py">gfstest.py</a> client/server test module<br>
<a href="gfinstall.py">gfinstall.py</a> installation script<br>
<a href="gftest.py">gftest.py</a> test suite<br>
<a href="sqlbind.py">sqlbind.py</a> grammar bindings<br>
<a href="sqlgen.py">sqlgen.py</a> grammar generation<br>
<a href="sqlgram.py">sqlgram.py</a> SQL grammar<br>
<a href="sqlgtest.py">sqlgtest.py</a> grammar tests<br>
<a href="sqlsem.py">sqlsem.py</a> main semantic objects<br>
<a href="sqlmod.py">sqlmod.py</a> update semantic objects<br>
<a href="gfdb0.py">gfdb0.py</a> data and file archiving<br>
<a href="gfintrospect.py">gfintrospect.py</a> introspection and remote table support<br>
<a href="remotetest.py">remotetest.py</a> remote table implementation demo and test</a><br>
<a href="relalg.py">relalg.py</a> relational algebra interpreter (toy)<br>
<a href="kjbuckets0.py">kjbuckets0.py</a> base data structures
(python version)<br>
</td>
<td valign="top">
These are the core files to the
<a href="gadfly.html">
Gadfly SQL database engine
</a>
A relational database query engine that supports
the Structured Query Language (SQL), implemented entirely
in Python (with optional builtin support from the
<a href="../kjbuckets">
kjbuckets builtin data structure accelerator</a>).
<hr>
Gadfly has been tested on Windows 95, Windows NT, Linux, and Unix
(solaris),
and it should run anywhere that Python runs (bebox,
maybe Palm Pilot/WinCE eventually...)
<hr>
Supports <a href="gfrecover.html">transactions and recovery</a>
<hr>
Supports <a href="gfSQL.html">a LARGE subset of SQL</a>
<hr>
Supports <a href="server.html">client/server access
via TCP/IP sockets.</a>
<hr>
<em>Many thanks to users who have used Gadfly in previous
releases and who have offered suggestions and bug reports!</em>
</td>
</tr>
<caption align="top"><b>Gadfly source modules and documentation</b></caption>
</table>
<h2>kwParser, Python lint, IDL, etc...</h2>
kwParser is a parser generator for Python. It transforms an
abstract specification of a language grammar (for example the
CORBA Interface Definition Language) together with "interpretation
functions" that define the semantics of the language into a
compiler or translator or interpreter. In the case of CORBA IDL
a python program using kwParser could generate stubs and support
code (in Python or some other language) to talk to a CORBA interface.
<p>
The release given here has had some micro-optimizations (26 June 1997)
which with luck don't break anything. In particular regexen are
used more intelligently and the generation phase will use
<a href="../kjbuckets">kjbuckets</a>
if it's available.
</p>
This is moderately heavy computer science. Not for the timid.
<p>
<TABLE BORDER>
<TR>
<TD><a href="COPYRIGHT">COPYRIGHT</a>
</TD>
<td>Do what you like, just don't sue me (roughly translated).
</td>
</TR>
<TR>
<TD><a href="DLispShort.py">DLispShort.py</a>
</TD>
<td>A very simple example language specification.
</td>
</TR>
<TR>
<TD><a href="DumbLispGen.py">DumbLispGen.py</a>
</TD>
<td>Another example specification
</td>
</TR>
<TR>
<TD><a href="arefize.py">arefize.py</a>
</TD>
<td>(not related: simple program used to generate this page)
</td>
</TR>
<TR>
<TD><a href="idl.py">idl.py</a>
</TD>
<td>A very complex example: CORBA IDL parser generator.
This is a good example of a complex grammar. The
interpretation functions and terminal regexes are
all stubbed.
</td>
</TR>
<TR>
<TD><a href="pygram.py">pygram.py</a>
</TD>
<td>A very complex example: The python grammar.
This module uses a hand written lexer to handle
Python's beautiful peculiarities. Used by kypylint.py.<br>
Look at the top of the module for editable parameters.
</td>
</TR>
<TR>
<TD><a href="kjpylint.py">kjpylint.py</a>
</TD>
<td>An attempt to use the pygram parser to do simple
checking on python source files. Reports references
not set, assignments not used, etcetera. not all warnings
indicate real problems of course. Used like this:
<pre>
% python kjpylint.py /home/app/arw/Python-1.5a1/Lib/SocketServer.py
setup
loading
now parsing
(verify_request) 'request' defined before 225 not used
(verify_request) 'self' defined before 225 not used
(verify_request) 'client_address' defined before 225 not used
(handle_error) 'request' defined before 245 not used
(handle_error) 'self' defined before 245 not used
(collect_children) 'status' defined before 293 not used
(setup) 'self' defined before 368 not used
(__del__) 'self' defined before 371 not used
(handle) 'self' defined before 374 not used
(finish) 'self' defined before 377 not used
(<module global>) '__version__' defined before 104 not used
269: (qref) 'max_packet_size' not defined in module?
</pre>
Here only the last line indicates a possible real bug
in SocketServer.py
<hr>
<em>Bugs/Features: <br>
Barfs on inconsistent indentation (ie space-tab is not the same as tab).
<br>
Grumpy about one line for loops and lambdas.<br>
For very weird cases may not parse strings correctly.<br>
See top of module for more info.<br>
Right now kjpylint might like to see an extra newline
at the end of the file. I'll look into this, sorry.</em>
<br>
Latest: Thu Jul 17 13:50:03 EDT 1997
</td>
</TR>
<TR>
<TD><a href="kjParseBuild.py">kjParseBuild.py</a>
</TD>
<td>The Parser generator module, used only to generate
the data structures required for parsing.
</td>
</TR>
<TR>
<TD><a href="kjParser.py">kjParser.py</a>
</TD>
<td>The Parser module, used both during the generation
phase and also after generation when the generated
parser is used.
</td>
</TR>
<TR>
<TD><a href="kwParsing.html">kwParsing.html</a>
</TD>
<td>HTML documentation for the package.
</td>
</TR>
<TR>
<TD><a href="kjSet.py">kjSet.py</a>
</TD>
<td>support module for parser generation.
(uses <a href="../kjbuckets">kjbuckets</a>
builtin if available, or uses a straight
Python implementation if not.)
</td>
</TR>
<TR>
<TD><a href="kwP.tar.gz">kwP.tar.gz</a>
</TD>
<td>The package (tarred, gzipped)
</td>
</TR>
<CAPTION ALIGN="top">kwParsing parser generator related files.
</CAPTION>
</TABLE>
<a href="mailto:arw@ifu.net">feedback</a><br>
<a href="../index.html">home</a>
</BODY>
</HTML>
=== Added File Packages/GadflyDA/gadfly/kjParseBuild.py === (1229/1329 lines abridged)
#
# python code for building a parser from a grammar
# Copyright Aaron Robert Watters, 1994
#
# BUGS:
# A bad grammar that has no derivations for
# the root nonterminal may cause a name error
# on the variable "GoodStartingPlace"
# this needs to be modified so the RULEGRAM is loaded from a
# compiled representation if available.
import string
import kjSet
import kjParser
import re
# import some constants
from kjParser import \
TERMFLAG, NOMATCHFLAG, MOVETOFLAG, REDUCEFLAG, TRANSFLAG, KEYFLAG, \
NONTERMFLAG, TERMFLAG, EOFFLAG, ENDOFFILETOKEN
PMODULE = kjParser.THISMODULE
# errors raised here
TokenError = "TokenError" # may happen on autogen with bad grammar
NotSLRError = "NotSLRError" # may happen for nonSLR grammar
# set this flag for regression testing at each load
RUNTESTS = 0
# set this flag to abort automatic generation on Errors
ABORTONERROR = 0
# token used to mark null productions
NULLTOKEN = (None,None)
# a derived FSM class, with closure computation methods defined
# (compilable FSMachine)
#
class CFSMachine(kjParser.FSMachine):
def __init__(self, nonterm):
kjParser.FSMachine.__init__(self, nonterm)
# return the epsilon closure of the FSM as a new FSM
#
# DoNullMap, if set, will map unexpected tokens to
# the "empty" state (usually creating a really big fsm)
[-=- -=- -=- 1229 lines omitted -=- -=- -=-]
RX = kjParser.ParseRule( X, [ oppar, Y, clpar ] )
RY = kjParser.ParseRule( Y, [] )
rl2 = [RX,RY]
rs2 = ruleset(X, rl2)
rs2.CompFirst()
rs2.CompFollow()
rs2.CompSLRNFA()
rs2.CompDFA()
rs2.SLRFixDFA()
DFA2 = rs2.DFA
ttt2 = dummy()
def TESTDFA2( STRING, DOREDUCTIONS = 1):
return TESTDFA( STRING, ttt2, DFA2, rl2, DOREDUCTIONS )
# the following grammar should fail to be slr
# (Aho,Ullman p. 213)
S = kjParser.nonterminal("S")
L = kjParser.nonterminal("L")
R = kjParser.nonterminal("R")
RS1 = kjParser.ParseRule( S, [L, equals, R] )
RS2 = kjParser.ParseRule( S, [R], echo )
RL1 = kjParser.ParseRule( L, [star, R])
RL2 = kjParser.ParseRule( L, [id])
RR1 = kjParser.ParseRule( R, [L] )
rs3 = ruleset(S, [RS1,RS2,RL1,RL2,RR1])
rs3.CompFirst()
rs3.CompFollow()
rs3.CompSLRNFA()
rs3.CompDFA()
#rs3.SLRFixDFA() # should fail and does.
# testing RULEGRAM
ObjG = NullCGrammar()
ObjG.Addterm("id","id",echo)
ObjG.Nonterms("T E Ep F Tp")
ObjG.Keywords("begin end")
ObjG.punct("+*()")
ObjG.comments(["--.*\n"])
# PROBLEM WITH COMMENTS???
Rulestr = """
## what a silly grammar!
T ::
@R One :: T >> begin E end
@R Three :: E >>
@R Two :: E >> E + T
@R Four :: E >> ( T )
"""
RL = RULEGRAM.DoParse1( Rulestr, ObjG )
=== Added File Packages/GadflyDA/gadfly/kjParser.py === (1210/1310 lines abridged)
#
# python for parser interpretation
# Copyright Aaron Robert Watters, 1994
#
# BUGS:
# Lexical error handling is not nice
# Parse error handling is not nice
#
# Lex analysis may be slow for big grammars
# Setting case sensitivity for keywords MUST happen BEFORE
# declaration of keywords.
import kjSet
import string
import re
import string
# set this flag for regression testing at each load
RUNTESTS = 0
# set this flag to enable warning for default reductions
WARNONDEFAULTS = 0
# some local constants
TERMFLAG = -1 # FLAG FOR TERMINAL
NOMATCHFLAG = -2 # FLAG FOR NO MATCH IN FSM
MOVETOFLAG = -3 # FLAG FOR "SHIFT" IN SN FSM
REDUCEFLAG = -4 # FLAG FOR REDUCTION IN FSM
TRANSFLAG = -5 # FLAG FOR TRANSIENT STATE IN FSM
KEYFLAG = -6 # FLAG FOR KEYWORD
NONTERMFLAG = -7 # FLAG FOR NONTERMINAL
TERMFLAG = -8 # FLAG FOR TERMINAL
EOFFLAG = "*" # FLAG for End of file
# set this string to the Module name (filename)
# used for dumping reconstructable objects
THISMODULE = "kjParser"
# regular expression for matching whitespace
WHITERE = "["+string.whitespace+"]+"
WHITEREGEX = re.compile(WHITERE)
# local errors
LexTokenError = "LexTokenError" # may happen on bad string
UnkTermError = "UnkTermError" # ditto
BadPunctError= "BadPunctError" # if try to make whitespace a punct
ParseInitError = "ParseInitError" # shouldn't happen?
#EOFError # may happen on bad string
FlowError = "FlowError" # shouldn't happen!!! (bug)
[-=- -=- -=- 1210 lines omitted -=- -=- -=-]
for tokenindex in range(len(tokens)):
(kind,name) = tokens[tokenindex]
if kind == KEYFLAG:
tokens[tokenindex] = LexD.keyword(name)
elif not kind in [TERMFLAG, NONTERMFLAG]:
raise FlowError, "unknown token type"
# not needed
self.tokens = tokens
def MakeRules(self):
Grammar = self.Gram
Grammar.DFA.root_nonTerminal = self.Root
NameIndex = Grammar.RuleNameToIndex
RuleTuples = self.RuleTups
nRules = len(RuleTuples)
RuleList = [None] * nRules
for index in range(nRules):
(Name, Components) = RuleTuples[index]
rule = apply(ParseRule, Components)
rule.Name = Name
RuleList[index] = rule
NameIndex[Name] = index
Grammar.RuleL = RuleList
def MakeTransitions(self):
Grammar = self.Gram
DFA = Grammar.DFA
StateTokenMap = DFA.StateTokenMap
tokens = self.tokens
# record the state number
DFA.maxState = self.MaxStates
# this is historical, unfortunately... CLEAN IT UP SOMEDAY!
# THE DFA.States DICT IS NOT NEEDED (?) (here)
for state in range(1, self.MaxStates+1):
DFA.States[state] = [TRANSFLAG]
# record the reductions
for (fromState, TokenIndex, rulenum) in self.reducts:
DFA.SetReduction(fromState, tokens[TokenIndex], rulenum)
# record the transitions
for (fromState, TokenIndex, ToState) in self.moveTos:
DFA.SetMap(fromState, tokens[TokenIndex], ToState)
def Cleanup(self):
Grammar = self.Gram
Grammar.CleanUp()
################# FOLLOWING CODE IS FOR REGRESSION TESTING ONLY
################# DELETE IT IF YOU WANT/NEED
#### All tests for this module deleted, since
#### ParseBuild module tests are sufficient.
=== Added File Packages/GadflyDA/gadfly/kjSet.py ===
#
# sets implemented using mappings
# Copyright Aaron Robert Watters, 1994
#
# these only work for "immutable" elements.
# probably not terribly efficient, but easy to implement
# and not as slow as concievably possible.
def NewSet(Sequence):
Result = {}
for Elt in Sequence:
Result[Elt] = 1
return Result
def Empty(Set):
if Set == {}:
return 1
else:
return 0
def get_elts(Set):
return Set.keys()
def member(Elt,Set):
return Set.has_key(Elt)
# in place mutators:
# returns if no change otherwise 1
def addMember(Elt,Set):
change = 0
if not Set.has_key(Elt):
Set[Elt] = 1
change = 1
return change
def Augment(Set, OtherSet):
change = 0
for Elt in OtherSet.keys():
if not Set.has_key(Elt):
Set[Elt] = 1
change = 1
return change
def Mask(Set, OtherSet):
change = 0
for Elt in OtherSet.keys():
if Set.has_key(Elt):
del Set[Elt]
change = 1
return change
# side effect free functions
def Intersection(Set1, Set2):
Result = {}
for Elt in Set1.keys():
if Set2.has_key(Elt):
Result[Elt] = 1
return Result
def Difference(Set1, Set2):
Result = {}
for Elt in Set1.keys():
if not Set2.has_key(Elt):
Result[Elt] = 1
return Result
def Union(Set1,Set2):
Result = {}
Augment(Result,Set1)
Augment(Result,Set2)
return Result
def Subset(Set1,Set2):
Result = 1
for Elt in Set1.keys():
if not Set2.has_key(Elt):
Result = 0
return Result # nonlocal
return Result
def Same(Set1,Set2):
if Subset(Set1,Set2) and Subset(Set2,Set1):
return 1
else:
return 0
# directed graphs as Dictionaries of Sets
# also only works for immutable nodes
def NewDG(pairlist):
Result = {}
for (source,dest) in pairlist:
AddArc(Result, source, dest)
return Result
def GetPairs(Graph):
result = []
Sources = Graph.keys()
for S in Sources:
Dests = get_elts( Graph[S] )
ThesePairs = [None] * len(Dests)
for i in range(0,len(Dests)):
D = Dests[i]
ThesePairs[i] = (S, D)
result = result + ThesePairs
return result
def AddArc(Graph, Source, Dest):
change = 0
if Graph.has_key(Source):
Adjacent = Graph[Source]
if not member(Dest,Adjacent):
addMember(Dest,Adjacent)
change = 1
else:
Graph[Source] = NewSet( [ Dest ] )
change = 1
return change
def Neighbors(Graph,Source):
if Graph.has_key(Source):
return get_elts(Graph[Source])
else:
return []
def HasArc(Graph, Source, Dest):
result = 0
if Graph.has_key(Source) and member(Dest, Graph[Source]):
result = 1
return result
def Sources(Graph):
return Graph.keys()
# when G1, G2 and G3 are different graphs this results in
# G1 = G1 U ( G2 o G3 )
# If G1 is identical to one of G2,G3 the result is somewhat
# nondeterministic (depends on dictionary implementation).
# However, guaranteed that AddComposition(G,G,G) returns
# G1 U (G1 o G1) <= G <= TC(G1)
# where G1 is G's original value and TC(G1) is its transitive closure
# hence this function can be used for brute force transitive closure
#
def AddComposition(G1, G2, G3):
change = 0
for G2Source in Sources(G2):
for Middle in Neighbors(G2,G2Source):
for G3Dest in Neighbors(G3, Middle):
if not HasArc(G1, G2Source, G3Dest):
change = 1
AddArc(G1, G2Source, G3Dest)
return change
# in place transitive closure of a graph
def TransClose(Graph):
change = AddComposition(Graph, Graph, Graph)
somechange = change
while change:
change = AddComposition(Graph, Graph, Graph)
if not somechange:
somechange = change
return somechange
########### SQueue stuff
#
# A GrabBag should be used to hold objects temporarily for future
# use. You can put things in and take them out, with autodelete
# that's all!
# make a new baggy with nothing in it
# BG[0] is insert cursor BG[1] is delete cursor, others are elts
#
OLD = 1
NEW = 0
START = 2
def NewBG():
B = [None]*8 #default size
B[OLD] = START
B[NEW] = START
return B
def BGempty(B):
# other ops must maintain this: old == new iff empty
return B[OLD] == B[NEW]
# may return new, larger structure
# must be used with assignment... B = BGadd(e,B)
def BGadd(elt, B):
cursor = B[NEW]
oldlen = len(B)
# look for an available position
while B[cursor] != None:
cursor = cursor+1
if cursor >= oldlen: cursor = START
if cursor == B[NEW]: #back to beginning
break
# resize if wrapped
if B[cursor] != None:
B = B + [None] * oldlen
cursor = oldlen
B[OLD] = START
if B[cursor] != None:
raise IndexError, "can't insert?"
# add the elt
B[cursor] = (elt,)
B[NEW] = cursor
# B nonempty so OLD and NEW should differ.
if B[OLD] == cursor:
B[NEW] = cursor + 1
if B[NEW]<=len(B): B[NEW] = START
return B
def BGgetdel(B):
# find something to delete:
cursor = B[OLD]
blen = len(B)
while B[cursor]==None:
cursor = cursor+1
if cursor>=blen: cursor = START
if cursor == B[OLD]: break # wrapped
if B[cursor] == None:
raise IndexError, "delete from empty grabbag(?)"
# test to see if bag is empty (position cursor2 at nonempty slot)
cursor2 = cursor+1
if cursor2>=blen: cursor2 = START
while B[cursor2]==None:
cursor2 = cursor2+1
if cursor2>=blen: cursor2 = START
# since B[cursor] not yet deleted while will terminate
# get and delete the elt
(result,) = B[cursor]
B[cursor] = None
# cursor == cursor2 iff bag is empty
B[OLD] = cursor2
if B[NEW] == cursor2: B[NEW] = cursor
return result
def BGtest(n):
B = NewBG()
rn = range(n)
rn2 = range(n-2)
for i in rn:
for j in rn:
B = BGadd( (i,j), B)
B = BGadd( (j,i), B)
x = BGgetdel(B)
for j in rn2:
y = BGgetdel(B)
print (i, x, y)
return B
=== Added File Packages/GadflyDA/gadfly/kjbuckets0.py === (950/1050 lines abridged)
# kjbuckets in pure python
### needs more thorough testing!
#import sys # for debug
def kjtabletest(x):
#print "kjtabletest"
try:
return x.is_kjtable
except:
return 0
unhashable = "unhashable key error"
class kjGraph:
is_kjtable = 1
def __init__(self, *args):
#print "kjGraph.__init__", args
key_to_list = self.key_to_list = {}
self.dirty = 0
self.hashed = None
#print args
if args:
if len(args)>1:
raise ValueError, "only 1 or 0 argument supported"
from types import IntType, ListType, TupleType
arg = args[0]
targ = type(arg)
test = key_to_list.has_key
if type(arg) is IntType:
return # ignore int initializer (presize not implemented)
elif type(arg) is ListType or type(arg) is TupleType:
for (x,y) in arg:
if test(x):
key_to_list[x].append(y)
else:
key_to_list[x] = [y]
return
aclass = arg.__class__
if aclass is kjGraph:
aktl = arg.key_to_list
for k in aktl.keys():
key_to_list[k] = aktl[k][:]
return
if aclass is kjDict or aclass is kjSet:
adict = arg.dict
[-=- -=- -=- 950 lines omitted -=- -=- -=-]
raise "trivial cmp fails", X
if not X:
raise "nonzero fails", X
if X is S:
if not S.member(0):
raise "huh 1?"
if S.member(123):
raise "huh 2?", S
S.add(999)
del S[1]
if not S.has_key(999):
raise "huh 3?", S
else:
print "values", X.values()
print "keys", X.keys()
print X, "inverted", ~X
if not X.member(0,1):
raise "member test fails (0,1)", X
print "adding to", X
X.add(999,888)
print "added", X
X.delete_arc(999,888)
print "deleted", X
if X.member(999,888):
raise "member test fails (999,888)", X
if X.has_key(999):
raise "has_key fails 999", X
if not X.has_key(0):
raise "has_key fails 0", X
for Y in ALL:
print "Y", Y
if (X!=S and Y!=S):
print "diff", X, Y
print "%s-%s=%s" % (X,Y,X-Y)
elif X==S:
D = kjSet(Y)
print "diff", X, D
print "%s-%s=%s" % (X,D,X-D)
print "%s+%s=%s" % (X,Y,X+Y)
print "%s&%s=%s" % (X,Y,X&Y)
print "%s*%s=%s" % (X,Y,X*Y)
x,y = cmp(X,Y), cmp(Y,X)
if x!=-y: raise "bad cmp!", (X, Y)
print "cmp(X,Y), -cmp(Y,X)", x,-y
print "X.subset(Y)", X.subset(Y)
=== Added File Packages/GadflyDA/gadfly/kjpylint.py === (500/600 lines abridged)
#!/usr/local/bin/python
"""python lint using kwParsing
The goal of this module/filter is to help find
programming errors in python source files.
As a filter use thusly:
% python kjpylint.py source_file.py
As an internal tool use like this:
import kjpylint
(pyg, context) = kjpylint.setup()
kjpylint.lint(data, pyg, context)
where data is the text of a python program.
You can build your own context structure by
subclassing GlobalContext, and redefining
GlobalContext.complain(string) for example.
You could do a lot more than that too...
Also, to lint all *.py files recursively contained
in a directory hierarchy use
kjpylint.lintdir("/usr/local/lib/python") # for example
FEATURES:
Lint expects
1) a newline or two at the end of the data;
2) consistent indenting (and inconsistency may be invisible)
[eg " \t" and "\t" are not the same indent
to Lint, but Python sees them the same.]
If (1) or (2) are not satisfied Lint will raise
an exception.
Buglets: lambdas and for loops on one line generate
extraneous warnings.
Notes:
======
The lint process works, in outline, like this.
Scan over a python program
x = 1
def f(a):
a = x
[-=- -=- -=- 500 lines omitted -=- -=- -=-]
class x(y,z):
'''
a doc string
blah
'''
def test(this, that):
w = that+this+x, n
x = 1
return w
"""
def go():
import sys
try:
file = sys.argv[1]
except IndexError:
print "required input file missing, defaulting to test string"
data = teststring
else:
data = open(file).read()
print "setup"
(pyg, context) = setup()
print "now parsing"
lint(data, pyg, context)
def setup():
global pyg, context
import pygram
pyg = pygram.unMarshalpygram()
BindRules(pyg)
context = globalContext(pyg.LexD)
return (pyg, context)
def lint(data, pygin=None, contextin=None):
if pygin is None: pygin = pyg
if contextin is None: contextin = context
pygin.DoParse1(data, contextin)
def lintdir(directory_name):
"""lint all files recursively in directory"""
from find import find
print "\n\nrecursively linting %s\n\n" % directory_name
(pyg, context) = setup()
python_files = find("*.py", directory_name)
for x in python_files:
print "\n\n [ %s ]\n\n" % x
lint( open(x).read(), pyg, context )
print "\014"
if __name__=="__main__": go()
=== Added File Packages/GadflyDA/gadfly/kwParsing.html === (985/1085 lines abridged)
<html>
<head>
<title>A parser generator in Python</title>
</head>
<body bgcolor="#ffffff">
<center>
<h1>A parser generator in Python: kwParsing</h1>
<blockquote>
Aaron Watters<br>
</blockquote>
<blockquote>
This is the documentation for the <strong>kjParsing</strong> package,
an experimental parser generator implemented in Python which generates
parsers implemented in Python.
It won't serve as a complete reference on programming
language syntax and interpretation, but it will review
terminology for the knowledgable and I hope it will pique
the interest of the less experienced.
</blockquote>
</center>
<h2>Introduction</h2>
<p>
The <code> kjParsing</code> package is a parser generator written
in Python which generates parsers for use in Python.
<p>
These modules and their documentation and demo files
may be of use for classes on parsing, compiling, or
formal languages, and may also be helpful to people
who like to create experimental interpreters or translators
or compilers.
<p>
The package consists of three Python modules:
<code> kjParser, kjParseBuild,</code> and <code> kjSet</code>. Together these
modules are called the <code> kjParsing</code> package.
The package also includes some documentation and demo
files and a <code> COPYRIGHT</code> file which explains the
conditions for copying and propagating this code
and the fact that the author assumes no responsibility
for any difficulties resulting from the use of this
package by anyone (including himself).
<h2>What a Parser Does</h2>
Parsers can be part of a lot of different things:
compilers, interpreters, translators, or code generators,
among others. Nevertheless, at an abstract level parsers
[-=- -=- -=- 985 lines omitted -=- -=- -=-]
This is normally resolved by informing
the parser generator to prefer one binding or the other.
No method for providing a preference is implemented here, yet.
Let me know if you need such a method or if you have any suggestions.
<p>
Keywords of the meta-grammar cannot name tokens of
the object grammar (see footnote above).
<p>
If you want keywords to be recognized without case
sensitivity you must declare <code> G.SetCaseSensitivity(0)</code>
before any keyword declarations.
<p>
Name and regular expression collisions are not always
checked and reported. If you name two rules the same,
for example, you may get undefined behavior.
<p>
The lexical analysis implementation is not as fast as it
could be (of course).
It also sees all white space as a
`single space'
so, for example, if indentation is significant in your grammar
(as in Python) you'll need a different lexical analyzer.
Also if <code> x=+y</code> means something different from
<code> x = + y</code> (as it did in the original C, I believe)
you may have trouble. Happily the lexical component can
be easily ``plug replaced'' by another implementation if needed.
<p>
Also, the system currently only handles SLR grammars (as defined
by Aho and Ullman), as mentioned above. If you get a
<code> NonSLRError</code> during grammar compilation you need a better
parser generator. I may provide one, if I have motivation and time.
<p>
I know of no outright bugs. Trust me, they're there. Please
find them for me and tell me about them. I'm not a big
expert on parsing so I'm sure I've made some errors, particularly
at the lexical level.
<h2>Further Reading</h2>
<p>
A standard reference for parsing and compiler, interpreter,
and translator implementation is Principles of Compiler
Design, by Aho and Ullman (Addison Wesley).
<p>
<a href="mailto:arw@ifu.net">feedback</a><br>
<a href="../index.html">home</a><br>
<a href="index.html">Gadfly home</a>
</body>
</html>
=== Added File Packages/GadflyDA/gadfly/pygram.py === (894/994 lines abridged)
# rules for python
# based on grammar given in Programming Python by Mark Lutz
# EDIT THIS: THE DIRECTORY IN WHICH TO MARSHAL THE
# GRAMMAR DATA STRUCTURES.
#
ARCHIVE = "."
marshalfilename = ARCHIVE + "/pygram.mar"
pyrules = """
all ::
## input terminates with "fake" dedent (forces read of all file)
@R all1 :: all >> file_input DEDENT
## 1 term newline
##@R lead_blank :: file_input >> NEWLINE file_input
@R top_stmt :: file_input >> file_input stmt
@R file_input :: file_input >> stmt
## 2
@R simple :: stmt >> simple_stmt
@R compound :: stmt >> compound_stmt
## 3 punct ; term NEWLINE
@R one_small :: simple_stmt >> small_stmt NEWLINE
@R more_small :: simple_stmt >> small_stmt ; simple_stmt
@R small_semi :: simple_stmt >> small_stmt ; NEWLINE
## 4 kw pass
@R smexpr :: small_stmt >> expr_stmt
@R smassn :: small_stmt >> assn
@R smprint :: small_stmt >> print_stmt
@R smdel :: small_stmt >> del_stmt
@R smpass :: small_stmt >> pass
@R smflow :: small_stmt >> flow_stmt
@R smimport :: small_stmt >> import_stmt
@R smglobal :: small_stmt >> global_stmt
## access ignored
@R smexec :: small_stmt >> exec_stmt
## 5
@R cmif :: compound_stmt >> if_stmt
@R cmwhile :: compound_stmt >> while_stmt
[-=- -=- -=- 894 lines omitted -=- -=- -=-]
d = {}
for i in range(10): d[i] = i
'''
def test(c,s):
return "this"
while not done:
print done
break
list = [1,2,3]
# comment
return 5
n,x = 89 >> 90 + 6 / 7 % x + z << 6 + 2 ** 8
if x==5:
while y:
for i in range(6):
raise SystemError, "oops"
"""
#teststring ="""\
## comment
#if x in y: print z
#elif 1: print w
#"""
'''
teststring="""
exec "print 1"
"""
'''
def test(grammar, context=None, teststring=teststring):
from time import time
now = time()
x = grammar.DoParse1(teststring, context)
elapsed = time()-now
print x
print elapsed
return x
regen = 0
dotest = 0
if __name__ == "__main__" :
if regen: GrammarBuild()
unMarshalpygram()
=== Added File Packages/GadflyDA/gadfly/relalg.py === (427/527 lines abridged)
"""Simple relational algebra interpreter.
usage:
To make the grammar
python relalg.py make
To run some relatoinal algebra expressions
python relalg.py < expressions_file
"""
# EDIT INSTALLDIR TO BE ABLE TO LOAD UNDER ANY CWD
INSTALLDIR = "."
## simple relational algebra using only the equality predicate
## note: string values cannot contain ;
## statement sequencing using ; handled at higher level
relalg_rules = """
statement ::
@R statementassn :: statement >> assignment
@R statementexpr :: statement >> rexpr
@R assignment1 :: assignment >> name = rexpr
@R assignmentn :: assignment >> name = assignment
@R union :: rexpr >> rexpr U rterm
@R rterm :: rexpr >> rterm
@R minus :: rexpr >> rexpr - rterm
@R intersect :: rterm >> rterm intersect rfactor
@R join :: rterm >> rterm join rfactor
@R rfactor :: rterm >> rfactor
@R projection :: rfactor >> projection [ names ] rfactor
@R names0 :: names >>
[-=- -=- -=- 427 lines omitted -=- -=- -=-]
def reloadrelalg(filename=MARSHALFILE):
import kjParser
filename = INSTALLDIR+"/"+filename
infile = open(filename, "rb")
SQLG = kjParser.UnMarshalGram(infile)
infile.close()
DeclareTerminals(SQLG)
BindRules(SQLG)
return SQLG
def runfile(f):
from string import split, join
ragram = reloadrelalg()
context = {}
#f = open(filename, "r")
data = f.read()
#f.close()
from string import split, strip
commands = split(data, ";")
for c in commands:
if not strip(c): continue
print " COMMAND:"
data = str(c)
pdata = " "+join(split(c, "\n"), "\n ")
print pdata
test = ragram.DoParse1(c, context)
print
# c:\python\python relalg.py ratest.txt
if __name__=="__main__":
try:
done = 0
import sys
argv = sys.argv
if len(argv)>1:
command = argv[1]
if command=="make":
print "building relational algebra grammar"
Buildrelalg()
done = 1
else:
runfile(sys.stdin)
done = 1
finally:
if not done:
print __doc__
=== Added File Packages/GadflyDA/gadfly/remotetest.py ===
"""Demonstration of the Remote View protocol for adding
specially implemented Views in an application."""
from gadfly import gadfly
# create the database
g = gadfly()
g.startup("dbtest", "dbtest") # assume directory "dbtest" exists
# define a remote view class
import gfintrospect
class myTable(gfintrospect.RemoteView):
"""A remote view must define self.column_names
to return a (fixed) list of string column names and
self.listing() to return a possibly varying
list of row values. If there is a single column
the listing() list must return a list of values,
but for multiple columns it must return a list
of tuples with one entry for each column.
The remote view implementation may optionally
redefine __init__ also, please see gfintrospect.py
"""
# static: don't reconstruct internal structure for each query
# for more interesting views static will generally be 0
static = 1
def __init__(self, column_names=None, rowlist=None):
"""do whatever needed for initialization"""
if column_names is None:
column_names = ['a', 'b', 'c']
if rowlist is None:
rowlist = [(1,2,3), (4,5,6), (7,8,9)]
self.column_names = column_names
self.rowlist = rowlist
def listing(self):
"""return list of tuples of right sizes to match column_names.
for more interesting views this will do something more
complex ;).
"""
return self.rowlist
# create a table using default cols and rows
### Python code adding ANY remote views must be EXECUTED
### EACH TIME THE DATABASE LOADS!
g.add_remote_view("test", myTable())
# create a table using specified cols and rows
# NOTE: for single column give list of values
# NOT list of tuples of values!
g.add_remote_view("test2", myTable(["x"], [1,6,7]))
print g.database
c = g.cursor()
c.execute("select * from test")
print "test::"
print c.pp()
print
c.execute("select * from test2")
print "test2::"
print c.pp()
print
c.execute("select * from test, test2 where x=a")
print "join"
print c.pp()
print
g.add_remote_view("test3", myTable(["z", "w"], [(2,3), (7,8), (4,6)]))
c.execute("select * from test3")
print "test3::"
print c.pp()
print
c.execute(
"select * from test, test2, test3 where x=a and z=b and w=c")
print "join 2::"
print c.pp()
print
=== Added File Packages/GadflyDA/gadfly/server.html ===
<html>
<head>
<title>Gadfly: server operations</title>
</head>
<body bgcolor="#ffffff">
<table><tr><td>
<img src="gadfly.JPG">
</td><td>
<h1>Gadfly Server Operations</h1>
</td></tr></table>
To permit multiple processes to access and modify a
single database instance, and to reduce the overhead per process
of connecting to a Gadfly database a Gadfly database may be
run in server mode. A Gadfly server can use a DBA
(data base administrator) configured
start-up script to set up optimized query accesses and certain
forms of security.
<p>
For example to startup a server for the
test database "test" in directory "dbtest" (created by gftest.py)
use:
<pre>
python gfserve.py 2222 test dbtest admin
</pre>
or to start up the same server with some non-priviledged
policies and some named prepared queries (as initialized
in gfstest.startup(...)) use
<pre>
python gfserve.py 2222 test dbtest admin gfstest
</pre>
In both cases the admin password for the server is "admin"
and the server runs on port 2222.
See the doc string for gfserve.py for more information on
the command line arguments.
<p>
Only one process should directly access a gadfly database at once
(not mediated by a server),
so if a server is running, no other server for that database
should be started and no other process should connect in "non-server"
mode to that database.
<h1>Motivation</h1>
There are several reasons to run a server: to allow multiple
processes to access the same database; to allow password protected
restricted access to the database by non-priviledged agents;
and to permit faster access to the database
by providing globally shared
prepared statements. Using a server also eliminates the need
to start up and load the database many times -- and startup
time could be considerable if the database is large.
<p>
For example I imagine that simple Gadfly servers may be of
use to implement database enabled CGI scripts, whereas the
"non-server" Gadfly will only run with CGI scripts that do not modify
the database, and the startup time for Gadfly might make those
scripts unacceptibly slow if the database is large. Furthermore,
by using the security features a Gadfly server could be configured
to allow restricted data distribution across a network without
compromising the integrity of the database.
<h1>Security</h1>
The primary goal of Gadfly server security is to prevent
accidental or malicious destruction of a database.
<p>
Security is arbitrated by policies. Policies have passwords
that are never transmitted in clear text. However, a "captured"
command could potentially be repeated by a hostile program
even without knowing the password. It is not a good idea to
run admin or other unrestricted commands on a network that may
have hostile parties sniffing the network. As with the rest
of the system I provide no guarantees, but for many purposes
the level of security provided may be acceptible. To be specific
passwords are used to generate md5 certificates for all server
accesses (please see gfsocket.py for implementation details).
<p>
A server always has
an "admin" policy that is permitted to shutdown, restart, or
force a checkpoint on the server. By default the admin
policy also has the ability to run arbitrary SQL statements
such as "drop table x". This ability can be disabled in
a startup function if needed.
<pre>
admin_policy.general_queries=0
</pre>
<p>
Other policies can be created that have very restricted access.
For example the following startup function initializes two
policies beyond the admin policy that can only access certain
tables in specific ways (from gfstest.py):
<pre>
def startup(admin_policy, connection, Server_instance):
"""example startup script.
add a policies test and test1 passwords same
test1 is allowed to query the likess table by name
test is allowed to update likes where drinker='nan'
also add prepared query dumpwork to admin_policy.
"""
from gfserve import Policy
admin_policy["dumpwork"] = "select * from work"
test1 = Policy("test1", "test1", connection, queries=0)
test = Policy("test", "test", connection, queries=0)
test1["qlike"] = "select * from likes where drinker=?"
test["updatenan"] = """
update likes
set beer=?, perday=?
where drinker='nan'
"""
test["getnan"] = """
select * from likes where drinker='nan'
"""
return {"test": test, "test1": test1}
</pre>
Please see the doc string for gfserve.py for more information
on creating startup functions.
<p>
A policy with queries disabled (queries=0) can only execute
named queries. By using such policies a DBA can configure
a server such that client programs can only read certain tables,
can only update certain rows of certain tables in certain ways,
and so forth.
<p>
Even policies with "unrestricted access" (queries=1)
can provide performance benefits if they have associated
named, prepared queries (like "dumpwork" above). At the moment
the SQL parser slows down gadfly a bit, and prepared queries
will only be parsed once for all clients. After the first
access subsequent accesses may be noticably faster (10x faster
in some cases), especially
if the server has the kjbuckets builtin C module. However,
with queries=1 the policy can execute any SQL statement.
<p>
<strong>NOTE: The server runs all accesses (once the complete
message has been read from the network) serially -- there is
no concurrent access permitted to
a Gadfly instance at this time. For this
reason a "large query" may cause the server to "freeze" and
delay other accesses.
</strong> Incomplete requests due to network delays or
other problems will not freeze the server, however (sockets
are polled using select.select).
<p>
<strong>NOTE: All server accesses run in "autocommit mode" at
this time. A successful access automatically triggers a database
commit (but an unsuccessful access will rollback).</strong>
<p>
As an optimization, however, checkpoints only occur occasionally,
once per a given number of accesses, configurable by setting:
<pre>
Server_instance.check_loop = 100
</pre>
<h1>Start up</h1>
Servers can be started from the command line using the gfserve.py
script interpretation
(as shown above)
or using gfserve.Server(...) from another
program. See the doc strings and source for gfserve.py and gfstest.py
for more information.
<h1>Shut down</h1>
Servers can be shut down from the command line interpretation of
gfclient.py or from another program using the gfclient(...) class
shutdown() method, but only using the admin policy with the admin
password. For example to shut down the server started above:
<pre>
python gfclient.py shutdown 2222 admin
</pre>
See the doc strings and source for gfserve.py
and gfstest.py
for more information.
<h1>Client Access</h1>
Client access to a gadfly server is similar to the normal
Python DB-SIG DBAPI
access to gadfly, except that it is sometimes faster and can
potentially
be run from any machine reachable on the network (if the client
program knows the password).
<p>
To access a gadfly server from a remote machine the only
python modules required (in addition to the standard libraries)
are gfclient.py and gfsocket.py.
<p>
Initialize a connection with a given "POLICY" with "PASSWORD"
to a running server
on "machine.domain.com" using port number 2222 with:
<pre>
from gfclient import gfclient
conn = gfclient("POLICY", 2222, "PASSWORD", "machine.domain.com")
</pre>
Note that policy names and passwords are case sensitive.
<p>
Queries and other statements are normally executed via cursors.
Obtain a cursor from a connection using:
<pre>
cursor = connection.cursor()
</pre>
Execute a statement in a cursor using:
<pre>
cursor.execute(statement)
</pre>
or to provide dynamic parameters:
<pre>
cursor.execute(statement, dynamic_parameters)
</pre>
For example
<pre>
cursor.execute("select * from work")
...
cursor.execute("select * from work where name=?", ("carla",))
</pre>
The dynamic parameters work the same as described in the
<a href="gadfly.html">the main gadfly documentation page</a>.
In particular INSERT VALUES can insert several rows at once
by using a list of tuples for the rows.
<p>
If there is any problem (bad policy name, bad password, server
not running, queries not allowed for this policy)
the execute will generate an exception.
<p>
To run a named/prepared query (initialized at startup) use
execute_prepared, which takes a prepared statement name
rather than a query string:
<pre>
cursor.execute_prepared("updatenan", ("rollingrock", 1))
...
cursor.execute_prepared("getnan")
</pre>
The execute_prepared method works just like the execute
method except that the "name" must be the name of a query initialized
by the startup(...) function at server startup.
<p><strong>
NOTE: by default any execution that sends or recieves "too much
data" will be aborted. Edit gfsocket.py (both on the client end
and on the server end if different) if you wish to disable this
sanity check feature.
<pre>
LEN_LIMIT=10e8
</pre></strong>
<p>
As with other dbapi cursors the results of a query can be
extracted as a list of tuples using (after execute):
<pre>
result_list = cursor.fetchall()
</pre>
The other fetches (fetchone and fetchmany) have not been
implemented yet (partially since they don't make much sense
in this context).
<p>
Both named and unnamed statements may be semicolon separated
sequences of several SQL statements, but if they are they will return
no results.
<h1>Implementation Comments</h1>
For your information the server/client interaction is much like
"finger" or "http" -- each client access is a separate TCP/Stream
connection where the client sends a request and the server sends
a response. After each access the connection is closed and the
next access generates a new connection.
I did it that way, because it was a simple and
robust strategy (witness the success of HTTP).
<p>
<strong>
Please note: Although I have attempted to provide a robust
implementation
for this software I do not guarantee its correctness. I hope
it will work well for you but I do not assume any legal
responsibility for problems anyone may have during use
of these programs.
</strong>
<p>
<a href="mailto:arw@ifu.net">feedback</a><br>
<a href="../index.html">home</a><br>
<a href="index.html">Gadfly home</a>
</Body>
</html>
=== Added File Packages/GadflyDA/gadfly/sql.mar ===
<Binary-ish file>
=== Added File Packages/GadflyDA/gadfly/sqlbind.py === (525/625 lines abridged)
"""rule bindings for sql grammar."""
def elt0(list, context):
"""return first member of reduction"""
return list[0]
def elt1(list, context):
"""return second member"""
return list[1]
def elt2(list, context):
return list[2]
def returnNone(list, context):
return None
def stat1(list, context):
"""return list of len 1 of statements"""
return list
#def statn(list, context):
# """return a list of statement reductions"""
# [stat, semi, statlist] = list
# statlist.insert(0, stat)
# return statlist
def thingcommalist(l, c):
[thing, comma, list] = l
list.insert(0, thing)
return list
def listcommathing(l, c):
[list, comma, thing] = l
list.append(thing)
return list
statn = thingcommalist
selstat = elt0
insstat = elt0
createtablestat = elt0
droptablestat = elt0
delstat = elt0
updatestat = elt0
createindexstat = elt0
dropindexstat = elt0
createviewstat = elt0
dropviewstat = elt0
# drop view statement stuff
def dropview(l, c):
[-=- -=- -=- 525 lines omitted -=- -=- -=-]
def selectn(list, context):
[ selectsubs, comma, select_sublist ] = list
(exp, name) = select_sublist
selectsubs.addbinding(name, exp)
return selectsubs
def selectit(list, context):
[exp] = list
return (exp, None) # no binding!
def selectname(list, context):
[exp, as, alias] = list
return (exp, alias)
colalias = elt0
#### do the bindings.
# note: all reduction function defs must precede this assign
VARS = vars()
class punter:
def __init__(self, name):
self.name = name
def __call__(self, list, context):
print "punt:", self.name, list
return list
class tracer:
def __init__(self, name, fn):
self.name = name
self.fn = fn
def __call__(self, list, context):
print self.name, list
return self.fn(list, context)
def BindRules(sqlg):
for name in sqlg.RuleNameToIndex.keys():
if VARS.has_key(name):
#print "binding", name
sqlg.Bind(name, VARS[name]) # nondebug
#sqlg.Bind(name, tracer(name, VARS[name]) ) # debug
else:
print "unbound", name
sqlg.Bind(name, punter(name))
return sqlg
=== Added File Packages/GadflyDA/gadfly/sqlgen.py ===
"""grammar generation stuff for sql.
This module does not bind any rule semantics, it
just generates the parser data structures.
"""
### interpretation functions and regexen for terminals
MARSHALFILE = "sql.mar"
import string
alphanum = string.letters+string.digits + "_"
userdefre = "[%s][%s]*" % (string.letters +"_", alphanum)
commentre = "--.*"
def userdeffn(str):
from string import upper
return upper(str)
charstre = "'[^']*'"
def charstfn(str):
return str[1:-1]
#numlitre = "[%s][%s\.]*" % (string.digits, alphanum) # not really...
digits = string.digits
# rely in python to filter out the good/bad/ugly
intre = "[%s][%s.jJ]*" % (digits,digits)
numlitre = "%s([Ee][+-]?%s)?" % (intre, intre)
def numlitfn(str):
"""Note: this is "safe" because regex
filters out dangerous things."""
return eval(str)
def DeclareTerminals(Grammar):
Grammar.Addterm("user_defined_name", userdefre, userdeffn)
Grammar.Addterm("character_string_literal", charstre, charstfn)
Grammar.Addterm("numeric_literal", numlitre, numlitfn)
def BuildSQL(filename=MARSHALFILE):
import kjParseBuild
from sqlgram import sqlrules, nonterms, keywords, puncts
SQLG = kjParseBuild.NullCGrammar()
SQLG.SetCaseSensitivity(0)
DeclareTerminals(SQLG)
SQLG.Keywords(keywords)
SQLG.punct(puncts)
SQLG.Nonterms(nonterms)
SQLG.comments([commentre])
# should add comments
SQLG.Declarerules(sqlrules)
print "working..."
SQLG.Compile()
print "testing"
from sqlgtest import test
for x in test:
print SQLG.DoParse1(x)
print "dumping to", filename
outfile = open(filename, "wb")
SQLG.MarshalDump(outfile)
outfile.close()
return SQLG
def reloadSQLG(filename=MARSHALFILE):
"""does not bind any interpretation functions."""
import kjParser
infile = open(filename, "rb")
SQLG = kjParser.UnMarshalGram(infile)
infile.close()
DeclareTerminals(SQLG)
return SQLG
def getSQL():
from sqlwhere import filename
return reloadSQLG(filename)
=== Added File Packages/GadflyDA/gadfly/sqlgram.py ===
# sql grammar, partial, based on ODBC 2.0 programmer's ref
## someday add subquery precedence to allow more general selects.
sqlrules = """
statement_list ::
@R stat1 :: statement_list >> statement
@R statn :: statement_list >> statement ; statement_list
@R dropindexstat :: statement >> drop_index_statement
@R createindexstat :: statement >> create_index_statement
@R selstat :: statement >> select_statement
@R insstat :: statement >> insert_statement
@R createtablestat :: statement >> create_table_statement
@R droptablestat :: statement >> drop_table_statement
@R delstat :: statement >> delete_statement_searched
@R updatestat :: statement >> update_statement_searched
@R createviewstat :: statement >> create_view_statement
@R dropviewstat :: statement >> drop_view_statement
## drop view statement
@R dropview :: drop_view_statement >> DROP VIEW user_defined_name
## create view statement
@R createview :: create_view_statement >>
CREATE VIEW user_defined_name optnamelist AS select_statement
@R optnamelist0 :: optnamelist >>
@R optnamelistn :: optnamelist >> ( namelist )
## drop index statement
@R dropindex :: drop_index_statement >> DROP INDEX user_defined_name
## create index statement
@R createindex :: create_index_statement >>
CREATE INDEX user_defined_name
ON user_defined_name
( namelist )
@R createuniqueindex :: create_index_statement >>
CREATE UNIQUE INDEX user_defined_name
ON user_defined_name
( namelist )
@R names1 :: namelist >> user_defined_name
@R namesn :: namelist >> namelist , user_defined_name
## update statement
@R update :: update_statement_searched >>
UPDATE user_defined_name
SET assns
optwhere
@R assn1 :: assns >> assn
@R assnn :: assns >> assns , assn
@R assn :: assn >> column_identifier = expression
#####
## delete statement
@R deletefrom :: delete_statement_searched >> DELETE FROM user_defined_name optwhere
## drop table
@R droptable :: drop_table_statement >> DROP TABLE user_defined_name
## create table statement ( restricted )
@R createtable :: create_table_statement >>
CREATE TABLE user_defined_name ( colelts )
@R colelts1 :: colelts >> colelt
@R coleltsn :: colelts >> colelts , colelt
@R coleltid :: colelt >> column_definition
@R coleltconstraint :: colelt >> column_constraint_definition
## column constraints deferred
@R coldef :: column_definition >>
column_identifier data_type optdefault optcolconstraints
## optdefault deferred
@R optdef0 :: optdefault >>
## optcolconstraint deferred
@R optcolconstr0 :: optcolconstraints >>
@R stringtype :: data_type >> character_string_type
@R exnumtype :: data_type >> exact_numeric_type
@R appnumtype :: data_type >> approximate_numeric_type
@R integer :: exact_numeric_type >> INTEGER
@R float :: approximate_numeric_type >> FLOAT
@R varchar :: character_string_type >> VARCHAR
@R varcharn :: character_string_type >> VARCHAR ( numeric_literal )
## insert statement
@R insert1 :: insert_statement >>
INSERT INTO table_name optcolids insert_spec
@R optcolids0 :: optcolids >>
@R optcolids1 :: optcolids >> ( colids )
@R colids1 :: colids >> column_identifier
@R colidsn :: colids >> colids , column_identifier
@R insert_values :: insert_spec >> VALUES ( litlist )
@R insert_query :: insert_spec >> sub_query
@R litlist1 :: litlist >> sliteral
@R litlistn :: litlist >> litlist , sliteral
@R sliteral0 :: sliteral >> literal
@R sliteralp :: sliteral >> + literal
## hack to permit complexes
@R sliterals :: sliteral >> sliteral + literal
@R sliterald :: sliteral >> sliteral - literal
@R sliteralm :: sliteral >> - literal
## select statement
@R subselect :: sub_query >>
SELECT alldistinct select_list
FROM table_reference_list
optwhere optgroup opthaving optunion
## @R psubselect :: sub_query >> ( sub_query )
@R selectx :: select_statement >>
sub_query
optorder_by
@R ad0 :: alldistinct >>
@R adall :: alldistinct >> ALL
@R addistinct :: alldistinct >> DISTINCT
@R where0 :: optwhere >>
@R where1 :: optwhere >> WHERE search_condition
@R group0 :: optgroup >>
@R group1 :: optgroup >> GROUP BY colnamelist
@R colnames1 :: colnamelist >> column_name
@R colnamesn :: colnamelist >> colnamelist , column_name
@R having0 :: opthaving >>
@R having1 :: opthaving >> HAVING search_condition
@R union0 :: optunion >>
@R union1 :: optunion >> UNION alldistinct sub_query
@R except1 :: optunion >> EXCEPT sub_query
@R intersect1 :: optunion >> INTERSECT sub_query
@R order0 :: optorder_by >>
@R order1 :: optorder_by >> ORDER BY sortspeclist
##@R orderby :: order_by_clause >> ORDER BY sortspeclist
@R sortspec1 :: sortspeclist >> sort_specification
@R sortspecn :: sortspeclist >> sortspeclist , sort_specification
## really, should be unsigned int
@R sortint :: sort_specification >> numeric_literal opt_ord
@R sortcol :: sort_specification >> column_name opt_ord
@R optord0 :: opt_ord >>
@R optordasc :: opt_ord >> ASC
@R optorddesc :: opt_ord >> DESC
## table reference list (nasty hack alert)
@R trl1 :: table_reference_list >> user_defined_name
@R trln :: table_reference_list >> user_defined_name , table_reference_list
@R trl1a :: table_reference_list >> user_defined_name user_defined_name
@R trlna :: table_reference_list >> user_defined_name user_defined_name , table_reference_list
@R trl1as :: table_reference_list >> user_defined_name AS user_defined_name
@R trlnas :: table_reference_list >> user_defined_name AS user_defined_name , table_reference_list
## select list
@R selectstar :: select_list >> *
@R selectsome :: select_list >> selectsubs
@R select1 :: selectsubs >> select_sublist
@R selectn :: selectsubs >> selectsubs , select_sublist
@R selectit :: select_sublist >> expression
@R selectname :: select_sublist >> expression AS column_alias
@R colalias :: column_alias >> user_defined_name
## search condition
@R search1 :: search_condition >> boolean_term
@R searchn :: search_condition >> boolean_term OR search_condition
@R bool1 :: boolean_term >> boolean_factor
@R booln :: boolean_term >> boolean_factor AND boolean_term
@R bf1 :: boolean_factor >> boolean_primary
@R notbf :: boolean_factor >> NOT boolean_primary
@R bp1 :: boolean_primary >> predicate
@R bps :: boolean_primary >> ( search_condition )
## predicate (simple for now!!!)
@R predicate1 :: predicate >> comparison_predicate
## comparison predicate (simple for now!!!)
@R predicateeq :: comparison_predicate >> expression = expression
@R predicatelt :: comparison_predicate >> expression < expression
@R predicategt :: comparison_predicate >> expression > expression
@R predicatele :: comparison_predicate >> expression < = expression
@R predicatege :: comparison_predicate >> expression > = expression
@R predicatene :: comparison_predicate >> expression < > expression
@R predbetween :: comparison_predicate >> expression BETWEEN expression AND expression
@R prednotbetween :: comparison_predicate >>
expression NOT BETWEEN expression AND expression
## exists predicate
@R predexists :: predicate >> exists_predicate
@R exists :: exists_predicate >> EXISTS ( sub_query )
## quantified predicate
@R predqeq :: predicate >> expression = allany ( sub_query )
@R predqne :: predicate >> expression < > allany ( sub_query )
@R predqlt :: predicate >> expression < allany ( sub_query )
@R predqgt :: predicate >> expression > allany ( sub_query )
@R predqle :: predicate >> expression < = allany ( sub_query )
@R predqge :: predicate >> expression > = allany ( sub_query )
@R nnall :: allany >> ALL
@R nnany :: allany >> ANY
## in predicate
@R predin :: predicate >> expression IN ( sub_query )
@R prednotin :: predicate >> expression NOT IN ( sub_query )
@R predinlits :: predicate >> expression IN ( litlist )
@R prednotinlits :: predicate >> expression NOT IN ( litlist )
## subquery expression
@R subqexpr :: expression >> ( sub_query )
## expression (simple for now!!!)
@R exp1 :: expression >> term
@R expplus :: expression >> expression + term
@R expminus :: expression >> expression - term
@R term1 :: term >> factor
@R termtimes :: term >> term * factor
@R termdiv :: term >> term / factor
@R factor1 :: factor >> primary
@R plusfactor :: factor >> + factor
@R minusfactor :: factor >> - factor
@R primary1 :: primary >> column_name
@R primarylit :: primary >> literal
@R primaryexp :: primary >> ( expression )
@R primaryset :: primary >> set_function_reference
@R stringlit :: literal >> character_string_literal
@R stringstring :: literal >> literal character_string_literal
@R numlit :: literal >> numeric_literal
## set functions (nasty hack!)
@R countstar :: set_function_reference >> COUNT ( * )
@R distinctcount :: set_function_reference >> COUNT ( DISTINCT expression )
@R allcount :: set_function_reference >> COUNT ( expression )
@R distinctset :: set_function_reference >> aggregate ( DISTINCT expression )
@R allset :: set_function_reference >> aggregate ( expression )
@R average :: aggregate >> AVG
##@R count :: aggregate >> COUNT
@R maximum :: aggregate >> MAX
@R minimum :: aggregate >> MIN
@R summation :: aggregate >> SUM
@R median :: aggregate >> MEDIAN
## dynamic parameter (varies quite a bit from ODBC spec)
@R dynamic :: literal >> ?
## column name
@R columnname1 :: column_name >> column_identifier
@R columnname2 :: column_name >> table_name . column_identifier
@R tablename1 :: table_name >> user_defined_name
@R columnid1 :: column_identifier >> user_defined_name
"""
nonterms = """
sliteral
exists_predicate set_function_reference aggregate
sortspeclist sort_specification opt_ord
drop_table_statement delete_statement_searched update_statement_searched
assns assn
insert_statement litlist colelt optcolconstraints optdefault
optcolids insert_spec create_table_statement
colids colelts column_constraint_definition
column_definition data_type character_string_type
exact_numeric_type approximate_numeric_type
expression term factor primary literal
comparison_predicate column_alias column_identifier table_name
boolean_term boolean_factor boolean_primary predicate
selectsubs expression alias sub_query
statement_list statement select_statement alldistinct subselect
select_list table_reference_list optwhere optgroup opthaving
order_by_clause select_sublist
optunion optorder_by search_condition colnamelist column_name
table_reference table_name create_index_statement namelist
drop_index_statement allany create_view_statement drop_view_statement
optnamelist
"""
keywords = """
INDEX ON ANY IN VIEW AS
EXCEPT INTERSECT
EXISTS AVG COUNT MAX MIN SUM MEDIAN
UPDATE DROP DELETE FROM SET
INSERT INTO VALUES CREATE TABLE INTEGER FLOAT VARCHAR
AND OR NOT
SELECT FROM WHERE HAVING GROUP BY UNION ALL DISTINCT AS ORDER
ASC DESC BETWEEN UNIQUE
"""
puncts = """.,*;=<>{}()?+-/"""
# terminals user_defined_name, character_string_literal,
# numeric_literal
=== Added File Packages/GadflyDA/gadfly/sqlgtest.py ===
"test parses for sql grammar"
test = [
"select a from x where b=c",
"select distinct x.a from x where x.b=c",
"select all a from x where b=c",
"select a from x, y where b=c or x.d=45",
"select a as k from x d, y as m where b=c",
"select 1 as n, a from x where b=c",
"select * from x",
"select a from x where b=c",
"select a from x where not b=c or d=1 and e=5",
"select a from x where a=1 and (x.b=3 or not b=c)",
"select -1 from x",
"select -1e6j from x",
"insert into table1 (a,b,c) values (-1e6+3j, -34e10, 56j)"
]
=== Added File Packages/GadflyDA/gadfly/sqlmod.py === (633/733 lines abridged)
"""Database modification statement semantics"""
import sqlsem
# ordering of ddef storage is important so, eg, index defs
# follow table defs.
class Ordered_DDF:
"""mixin for DDF statement sorting, subclass defines s.cmp(o)"""
def __cmp__(self, other):
try:
#print "comparing", self.name, other.name
try:
sc = self.__class__
oc = other.__class__
#print sc, oc
except:
#print "punting 1", -1
return -1
if sc in ddf_order and oc in ddf_order:
test = cmp(ddf_order.index(sc), ddf_order.index(oc))
#print "ddforder", test
if test: return test
return self.cmp(other)
else:
test = cmp(sc, oc)
#print "punting 2", test
return test
except:
#import sys
#print "exception!"
#print sys.exc_type, sys.exc_value
return -1
def __coerce__(self, other):
return (self, other)
def cmp(self, other):
"""redefine if no name field"""
return cmp(self.name, other.name)
CTFMT = """\
CREATE TABLE %s (
%s
)"""
class CreateTable(Ordered_DDF):
"""create table operation"""
def __init__(self, name, colelts):
self.name = name
[-=- -=- -=- 633 lines omitted -=- -=- -=-]
#print "bindings", dynbt.assns
return dynbt # ??
class InsertSubSelect(sqlsem.SimpleRecursive):
def __init__(self, subsel):
self.subsel = subsel
def initargs(self):
return (self.subsel,)
def __repr__(self):
return "[subsel] %s" % (self.subsel,)
def resultexps(self):
# get list of result bindings
subsel = self.subsel
atts = self.subsel.attributes()
# bind each as "result.name"
exps = []
from sqlsem import BoundAttribute
for a in atts:
exps.append( BoundAttribute("result", a) )
return exps # temp
def relbind(self, db):
subsel = self.subsel
self.subsel = subsel.relbind(db)
# do nothing with domain for now
#subsel_domain = subsel.domain()
return self
def eval(self, dyn=None):
subsel = self.subsel
subsel.uncache()
rel = subsel.eval(dyn)
tups = rel.rows()
from sqlsem import BoundTuple ### temp
from sqlsem import kjbuckets
kjDict = kjbuckets.kjDict
for i in xrange(len(tups)):
tupsi = tups[i]
new = kjDict()
for k in tupsi.keys():
new[ ("result", k) ] = tupsi[k]
tups[i] = new
return tups
# ordering for archiving datadefs
ddf_order = [CreateTable, CreateIndex, CreateView]
=== Added File Packages/GadflyDA/gadfly/sqlsem.py === (2850/2950 lines abridged)
""" sql semantics
"""
### trim unused methods.
### make assns use equivalence classes.
### maybe eventually implement disj-conj-eq optimizations
### note: for multithreading x.relbind(...) should ALWAYs return
### a fresh copy of structure (sometimes in-place now).
### note: binding of order by is dubious with archiving,
### should not bind IN PLACE, leave unbound elts alone!
### need to fix serialization/deserialization of btand and btor
###
# use kjbuckets builtin if available
try:
import kjbuckets
except ImportError:
import kjbuckets0
kjbuckets = kjbuckets0
Tuple = kjbuckets.kjDict
Graph = kjbuckets.kjGraph
Set = kjbuckets.kjSet
import sys, traceback
### debug
#sys.stderr = sys.stdin
# operations on simple tuples, mostly from kjbuckets
#def maketuple(thing):
# """try to make a tuple from thing.
# thing should be a dictionary or sequence of (name, value)
# or other tuple."""
# from types import DictType
# if type(thing)==DictType:
# return Tuple(thing.items() )
# else: return Tuple(thing)
def no_ints_nulls(list):
"""in place remove all ints, Nones from a list (for null handling)"""
tt = type
nn = None
from types import IntType
count = 0
for x in list:
[-=- -=- -=- 2850 lines omitted -=- -=- -=-]
allrows = (kjSet(assns) - kjSet(rows)).items()
return allrows
op = "EXCEPT"
class Parse_Context:
"""contextual information for parsing
p.param() returns a new sequence number for external parameter.
"""
# not serializable
parameter_index = 0
# no __init__ yet
def param(self):
temp = self.parameter_index
self.parameter_index = temp+1
return temp
def ndynamic(self):
return self.parameter_index
# update/delete/insert statements
import sqlmod
CreateTable = sqlmod.CreateTable
CreateIndex = sqlmod.CreateIndex
DropIndex = sqlmod.DropIndex
DropTable = sqlmod.DropTable
UpdateOp = sqlmod.UpdateOp
DeleteOp = sqlmod.DeleteOp
InsertOp = sqlmod.InsertOp
InsertValues = sqlmod.InsertValues
InsertSubSelect = sqlmod.InsertSubSelect
ColumnDef = sqlmod.ColumnDef
CreateView = sqlmod.CreateView
DropView = sqlmod.DropView
# update storage structures from gfdb0
import gfdb0
Add_Tuples = gfdb0.Add_Tuples
Erase_Tuples = gfdb0.Erase_Tuples
Reset_Tuples = gfdb0.Reset_Tuples
####### testing
# test helpers
#def tp(**kw):
# return maketuple(kw)
#def st(**kw):
# return BTPredicate(BoundTuple(r=kw))
=== Added File Packages/GadflyDA/gadfly/sqlwhere.py ===
'''this module indicates where the sql datastructures are marshalled
Auto generated on install: better not touch!
'''