[Zope-Checkins] CVS: Zope/lib/python/RestrictedPython/compiler - consts.py:1.2.16.1 future.py:1.2.16.1 misc.py:1.2.16.1 pyassem.py:1.2.16.3 pycodegen.py:1.4.2.1 symbols.py:1.3.8.2
Shane Hathaway
shane@digicool.com
Fri, 21 Dec 2001 14:13:41 -0500
Update of /cvs-repository/Zope/lib/python/RestrictedPython/compiler
In directory cvs.zope.org:/tmp/cvs-serv5071/compiler
Modified Files:
Tag: Zope-2_4-branch
consts.py future.py misc.py pyassem.py pycodegen.py symbols.py
Log Message:
Backported stacksize and other compiler bugfixes.
=== Zope/lib/python/RestrictedPython/compiler/consts.py 1.2 => 1.2.16.1 ===
-CO_VARARGS = 1
-CO_VARKEYWORDS = 2
-
# operation flags
OP_ASSIGN = 'OP_ASSIGN'
OP_DELETE = 'OP_DELETE'
@@ -12,3 +8,9 @@
SC_FREE = 3
SC_CELL = 4
SC_UNKNOWN = 5
+
+CO_OPTIMIZED = 0x0001
+CO_NEWLOCALS = 0x0002
+CO_VARARGS = 0x0004
+CO_VARKEYWORDS = 0x0008
+CO_NESTED = 0x0010
=== Zope/lib/python/RestrictedPython/compiler/future.py 1.2 => 1.2.16.1 ===
if __name__ == "__main__":
import sys
- from compiler import parseFile, walk
+ from transformer import parseFile
for file in sys.argv[1:]:
print file
=== Zope/lib/python/RestrictedPython/compiler/misc.py 1.2 => 1.2.16.1 ===
def top(self):
return self.stack[-1]
+ def __getitem__(self, index): # needed by visitContinue()
+ return self.stack[index]
+
+MANGLE_LEN = 256 # magic constant from compile.c
+
+def mangle(name, klass):
+ if not name.startswith('__'):
+ return name
+ if len(name) + 2 >= MANGLE_LEN:
+ return name
+ if name.endswith('__'):
+ return name
+ try:
+ i = 0
+ while klass[i] == '_':
+ i = i + 1
+ except IndexError:
+ return name
+ klass = klass[i:]
+
+ tlen = len(klass) + len(name)
+ if tlen > MANGLE_LEN:
+ klass = klass[:MANGLE_LEN-tlen]
+
+ return "_%s%s" % (klass, name)
+
=== Zope/lib/python/RestrictedPython/compiler/pyassem.py 1.2.16.2 => 1.2.16.3 ===
from __future__ import nested_scopes
+
import dis
import new
import string
@@ -8,6 +9,7 @@
import types
import misc
+from consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
def xxx_sort(l):
l = l[:]
@@ -54,7 +56,7 @@
# these edges to get the blocks emitted in the right order,
# however. :-( If a client needs to remove these edges, call
# pruneEdges().
-
+
self.current.addNext(block)
self.startBlock(block)
@@ -109,13 +111,13 @@
# XXX This is a total mess. There must be a better way to get
# the code blocks in the right order.
-
+
self.fixupOrderHonorNext(blocks, default_next)
self.fixupOrderForward(blocks, default_next)
def fixupOrderHonorNext(self, blocks, default_next):
"""Fix one problem with DFS.
-
+
The DFS uses child block, but doesn't know about the special
"next" block. As a result, the DFS can order blocks so that a
block isn't next to the right block for implicit control
@@ -195,19 +197,18 @@
chains.remove(c)
chains.insert(goes_before, c)
-
del blocks[:]
for c in chains:
for b in c:
blocks.append(b)
-
+
def getBlocks(self):
return self.blocks.elements()
def getRoot(self):
"""Return nodes appropriate for use with dominator"""
return self.entry
-
+
def getContainedGraphs(self):
l = []
for b in self.getBlocks():
@@ -246,7 +247,7 @@
def __str__(self):
insts = map(str, self.insts)
return "<block %s %d:\n%s>" % (self.label, self.bid,
- string.join(insts, '\n'))
+ string.join(insts, '\n'))
def emit(self, inst):
op = inst[0]
@@ -268,7 +269,7 @@
assert len(self.next) == 1, map(str, self.next)
_uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
- 'JUMP_ABSOLUTE', 'JUMP_FORWARD')
+ 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
def pruneNext(self):
"""Remove bogus edge for unconditional transfers
@@ -312,11 +313,6 @@
return contained
# flags for code objects
-CO_OPTIMIZED = 0x0001
-CO_NEWLOCALS = 0x0002
-CO_VARARGS = 0x0004
-CO_VARKEYWORDS = 0x0008
-CO_NESTED = 0x0010
# the FlowGraph is transformed in place; it exists in one of these states
RAW = "RAW"
@@ -327,15 +323,17 @@
class PyFlowGraph(FlowGraph):
super_init = FlowGraph.__init__
- def __init__(self, name, filename, args=(), optimized=0):
+ def __init__(self, name, filename, args=(), optimized=0, klass=None):
self.super_init()
self.name = name
+ assert isinstance(filename, types.StringType)
self.filename = filename
self.docstring = None
self.args = args # XXX
self.argcount = getArgCount(args)
+ self.klass = klass
if optimized:
- self.flags = CO_OPTIMIZED | CO_NEWLOCALS
+ self.flags = CO_OPTIMIZED | CO_NEWLOCALS
else:
self.flags = 0
self.consts = []
@@ -364,6 +362,10 @@
if flag == CO_VARARGS:
self.argcount = self.argcount - 1
+ def checkFlag(self, flag):
+ if self.flags & flag:
+ return 1
+
def setFreeVars(self, names):
self.freevars = list(names)
@@ -462,7 +464,6 @@
insts[i] = opname, offset
elif self.hasjabs.has_elt(opname):
insts[i] = opname, begin[inst[1]]
- self.stacksize = findDepth(self.insts)
self.stage = FLAT
hasjrel = misc.Set()
@@ -480,8 +481,7 @@
for i in range(len(self.insts)):
t = self.insts[i]
if len(t) == 2:
- opname = t[0]
- oparg = t[1]
+ opname, oparg = t
conv = self._converters.get(opname, None)
if conv:
self.insts[i] = opname, conv(self, oparg)
@@ -501,10 +501,16 @@
self.closure = self.cellvars + self.freevars
def _lookupName(self, name, list):
- """Return index of name in list, appending if necessary"""
+ """Return index of name in list, appending if necessary
+
+ This routine uses a list instead of a dictionary, because a
+ dictionary can't store two different keys if the keys have the
+ same value but different types, e.g. 2 and 2L. The compiler
+ must treat these two separately, so it does an explicit type
+ comparison before comparing the values.
+ """
t = type(name)
for i in range(len(list)):
- # must do a comparison on type first to prevent UnicodeErrors
if t == type(list[i]) and list[i] == name:
return i
end = len(list)
@@ -523,9 +529,15 @@
_convert_STORE_FAST = _convert_LOAD_FAST
_convert_DELETE_FAST = _convert_LOAD_FAST
+ def _convert_LOAD_NAME(self, arg):
+ if self.klass is None:
+ self._lookupName(arg, self.varnames)
+ return self._lookupName(arg, self.names)
+
def _convert_NAME(self, arg):
+ if self.klass is None:
+ self._lookupName(arg, self.varnames)
return self._lookupName(arg, self.names)
- _convert_LOAD_NAME = _convert_NAME
_convert_STORE_NAME = _convert_NAME
_convert_DELETE_NAME = _convert_NAME
_convert_IMPORT_NAME = _convert_NAME
@@ -557,7 +569,7 @@
for name, obj in locals().items():
if name[:9] == "_convert_":
opname = name[9:]
- _converters[opname] = obj
+ _converters[opname] = obj
del name, obj, opname
def makeByteCode(self):
@@ -587,13 +599,14 @@
def newCodeObject(self):
assert self.stage == DONE
- if self.flags == 0:
+ if (self.flags & CO_NEWLOCALS) == 0:
nlocals = 0
else:
nlocals = len(self.varnames)
argcount = self.argcount
if self.flags & CO_VARKEYWORDS:
argcount = argcount - 1
+
return new.code(argcount, nlocals, self.stacksize, self.flags,
self.lnotab.getCode(), self.getConsts(),
tuple(self.names), tuple(self.varnames),
@@ -613,7 +626,7 @@
elt = elt.getCode()
l.append(elt)
return tuple(l)
-
+
def isJump(opname):
if opname[:4] == 'JUMP':
return 1
@@ -707,17 +720,19 @@
def getTable(self):
return string.join(map(chr, self.lnotab), '')
-
+
class StackDepthTracker:
# XXX 1. need to keep track of stack depth on jumps
# XXX 2. at least partly as a result, this code is broken
- def findDepth(self, insts):
+ def findDepth(self, insts, debug=0):
depth = 0
maxDepth = 0
for i in insts:
opname = i[0]
- delta = self.effect.get(opname)
+ if debug:
+ print i,
+ delta = self.effect.get(opname, None)
if delta is not None:
depth = depth + delta
else:
@@ -732,10 +747,10 @@
meth = getattr(self, opname, None)
if meth is not None:
depth = depth + meth(i[1])
- if depth < 0:
- depth = 0
if depth > maxDepth:
maxDepth = depth
+ if debug:
+ print depth, maxDepth
return maxDepth
effect = {
@@ -756,9 +771,8 @@
'DELETE_SUBSCR': -2,
# PRINT_EXPR?
'PRINT_ITEM': -1,
- 'LOAD_LOCALS': 1,
'RETURN_VALUE': -1,
- 'EXEC_STMT': -2,
+ 'EXEC_STMT': -3,
'BUILD_CLASS': -2,
'STORE_NAME': -1,
'STORE_ATTR': -2,
@@ -774,23 +788,20 @@
# close enough...
'SETUP_EXCEPT': 3,
'SETUP_FINALLY': 3,
- 'FOR_ITER': 1,
+ 'FOR_LOOP': 1,
}
# use pattern match
patterns = [
('BINARY_', -1),
('LOAD_', 1),
]
-
- # special cases:
- # UNPACK_SEQUENCE, BUILD_TUPLE,
- # BUILD_LIST, CALL_FUNCTION, MAKE_FUNCTION, BUILD_SLICE
+
def UNPACK_SEQUENCE(self, count):
return count-1
def BUILD_TUPLE(self, count):
- return 1-count
+ return -count+1
def BUILD_LIST(self, count):
- return 1-count
+ return -count+1
def CALL_FUNCTION(self, argc):
hi, lo = divmod(argc, 256)
return -(lo + hi * 2)
@@ -802,6 +813,9 @@
return self.CALL_FUNCTION(argc)-2
def MAKE_FUNCTION(self, argc):
return -argc
+ def MAKE_CLOSURE(self, argc):
+ # XXX need to account for free variables too!
+ return -argc
def BUILD_SLICE(self, argc):
if argc == 2:
return -1
@@ -809,5 +823,5 @@
return -2
def DUP_TOPX(self, argc):
return argc
-
+
findDepth = StackDepthTracker().findDepth
=== Zope/lib/python/RestrictedPython/compiler/pycodegen.py 1.4 => 1.4.2.1 === (426/526 lines abridged)
import pyassem, misc, future, symbols
from consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL
-from pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\
- CO_NESTED, TupleArg
+from consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, CO_NESTED
+from pyassem import TupleArg
# Do we have Python 1.x or Python 2.x?
try:
@@ -30,15 +30,24 @@
(1,1) : "CALL_FUNCTION_VAR_KW",
}
+LOOP = 1
+EXCEPT = 2
+TRY_FINALLY = 3
+END_FINALLY = 4
+
def compile(filename, display=0):
f = open(filename)
buf = f.read()
f.close()
mod = Module(buf, filename)
- mod.compile(display)
- f = open(filename + "c", "wb")
- mod.dump(f)
- f.close()
+ try:
+ mod.compile(display)
+ except SyntaxError:
+ raise
+ else:
+ f = open(filename + "c", "wb")
+ mod.dump(f)
+ f.close()
class Module:
def __init__(self, source, filename):
@@ -117,6 +126,12 @@
def visitAssName(self, node):
self.names.add(node.name)
+def is_constant_false(node):
+ if isinstance(node, ast.Const):
+ if not node.value:
+ return 1
+ return 0
+
class CodeGenerator:
"""Defines basic code generator for Python bytecode
[-=- -=- -=- 426 lines omitted -=- -=- -=-]
+ __super_init = AbstractClassCode.__init__
+
+ def __init__(self, klass, scopes, filename):
+ self.scopes = scopes
+ self.scope = scopes[klass]
+ self.__super_init(klass, scopes, filename)
+ self.graph.setFreeVars(self.scope.get_free_vars())
+ self.graph.setCellVars(self.scope.get_cell_vars())
+
class NestedClassCodeGenerator(AbstractClassCode,
NestedScopeMixin,
NestedScopeCodeGenerator):
super_init = NestedScopeCodeGenerator.__init__ # call be other init
__super_init = AbstractClassCode.__init__
- def __init__(self, klass, filename, scopes):
+ def __init__(self, klass, scopes, filename):
+ assert isinstance(filename, types.StringType)
self.scopes = scopes
self.scope = scopes[klass]
- self.__super_init(klass, filename, scopes)
+ self.__super_init(klass, scopes, filename)
self.graph.setFreeVars(self.scope.get_free_vars())
self.graph.setCellVars(self.scope.get_cell_vars())
self.graph.setFlag(CO_NESTED)
@@ -1212,7 +1287,7 @@
def findOp(node):
"""Find the op (DELETE, LOAD, STORE) in an AssTuple tree"""
v = OpFinder()
- walk(node, v, 0)
+ walk(node, v, verbose=0)
return v.op
class OpFinder:
@@ -1224,6 +1299,7 @@
elif self.op != node.flags:
raise ValueError, "mixed ops in stmt"
visitAssAttr = visitAssName
+ visitSubscript = visitAssName
class Delegator:
"""Base class to support delegation for augmented assignment nodes
@@ -1253,6 +1329,7 @@
class AugSubscript(Delegator):
pass
+
wrapper = {
ast.Getattr: AugGetattr,
=== Zope/lib/python/RestrictedPython/compiler/symbols.py 1.3.8.1 => 1.3.8.2 ===
import ast
from consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_UNKNOWN
+from misc import mangle
import types
import sys
@@ -36,13 +37,7 @@
def mangle(self, name):
if self.klass is None:
return name
- if not name.startswith('__'):
- return name
- if len(name) + 2 >= MANGLE_LEN:
- return name
- if name.endswith('__'):
- return name
- return "_%s%s" % (self.klass, name)
+ return mangle(name, self.klass)
def add_def(self, name):
self.defs[self.mangle(name)] = 1
@@ -295,6 +290,27 @@
name = name[:i]
scope.add_def(asname or name)
+ def visitGlobal(self, node, scope):
+ for name in node.names:
+ scope.add_global(name)
+
+ def visitAssign(self, node, scope):
+ """Propagate assignment flag down to child nodes.
+
+ The Assign node doesn't itself contains the variables being
+ assigned to. Instead, the children in node.nodes are visited
+ with the assign flag set to true. When the names occur in
+ those nodes, they are marked as defs.
+
+ Some names that occur in an assignment target are not bound by
+ the assignment, e.g. a name occurring inside a slice. The
+ visitor handles these nodes specially; they do not propagate
+ the assign flag to their children.
+ """
+ for n in node.nodes:
+ self.visit(n, scope, 1)
+ self.visit(node.expr, scope)
+
def visitAssName(self, node, scope, assign=1):
scope.add_def(node.name)
@@ -306,6 +322,13 @@
for n in node.subs:
self.visit(n, scope, 0)
+ def visitSlice(self, node, scope, assign=0):
+ self.visit(node.expr, scope, 0)
+ if node.lower:
+ self.visit(node.lower, scope, 0)
+ if node.upper:
+ self.visit(node.upper, scope, 0)
+
def visitAugAssign(self, node, scope):
# If the LHS is a name, then this counts as assignment.
# Otherwise, it's just use.
@@ -314,15 +337,6 @@
self.visit(node.node, scope, 1) # XXX worry about this
self.visit(node.expr, scope)
- def visitAssign(self, node, scope):
- for n in node.nodes:
- self.visit(n, scope, 1)
- self.visit(node.expr, scope)
-
- def visitGlobal(self, node, scope):
- for name in node.names:
- scope.add_global(name)
-
# prune if statements if tests are false
_const_types = types.StringType, types.IntType, types.FloatType
@@ -348,7 +362,8 @@
if __name__ == "__main__":
import sys
- from compiler import parseFile, walk
+ from transformer import parseFile
+ from visitor import walk
import symtable
def get_names(syms):