mirrors/cpython
0
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2024-11-23 18:04:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Patch #1440601: Add col_offset attribute to AST nodes.

Browse Source
This commit is contained in:
Martin v. Löwis 2006-03-01 22:49:05 +00:00
parent 3ffa59b137
commit 49c5da1d88
16 changed files with 382 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Doc/lib/libast.tex
View File

@ -34,7 +34,13 @@ names of all child nodes.
Each instance of a concrete class has one attribute for each child node,
of the type as defined in the grammar. For example, \code{_ast.BinOp}
instances have an attribute \code{left} of type \code{_ast.expr}.
instances have an attribute \code{left} of type \code{_ast.expr}.
Instances of \code{_ast.expr} and \code{_ast.stmt} subclasses also
have lineno and col_offset attributes. The lineno is the line number
of source text (1 indexed so the first line is line 1) and the
col_offset is the utf8 byte offset of the first token that generated
the node. The utf8 offset is recorded because the parser uses utf8
internally.
If these attributes are marked as optional in the grammar (using a
question mark), the value might be \code{None}. If the attributes

117
Include/Python-ast.h
View File

@ -178,6 +178,7 @@ struct _stmt {
} v;
int lineno;
int col_offset;
};
struct _expr {
@ -288,6 +289,7 @@ struct _expr {
} v;
int lineno;
int col_offset;
};
struct _slice {
@ -346,68 +348,79 @@ mod_ty Interactive(asdl_seq * body, PyArena *arena);
mod_ty Expression(expr_ty body, PyArena *arena);
mod_ty Suite(asdl_seq * body, PyArena *arena);
stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorators, int lineno, PyArena *arena);
asdl_seq * decorators, int lineno, int col_offset, PyArena
*arena);
stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
lineno, PyArena *arena);
stmt_ty Return(expr_ty value, int lineno, PyArena *arena);
stmt_ty Delete(asdl_seq * targets, int lineno, PyArena *arena);
stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
stmt_ty Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
stmt_ty Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena);
stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset,
PyArena *arena);
stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno,
PyArena *arena);
stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena
*arena);
int col_offset, PyArena *arena);
stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int
col_offset, PyArena *arena);
stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
int lineno, PyArena *arena);
stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
PyArena *arena);
stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
PyArena *arena);
int lineno, int col_offset, PyArena *arena);
stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
col_offset, PyArena *arena);
stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
col_offset, PyArena *arena);
stmt_ty With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int
lineno, PyArena *arena);
stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena
*arena);
lineno, int col_offset, PyArena *arena);
stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int
col_offset, PyArena *arena);
stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
lineno, PyArena *arena);
stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena
*arena);
stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena);
stmt_ty Import(asdl_seq * names, int lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int
col_offset, PyArena *arena);
stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena
*arena);
stmt_ty Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
stmt_ty ImportFrom(identifier module, asdl_seq * names, int level, int lineno,
PyArena *arena);
stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena
*arena);
stmt_ty Global(asdl_seq * names, int lineno, PyArena *arena);
stmt_ty Expr(expr_ty value, int lineno, PyArena *arena);
stmt_ty Pass(int lineno, PyArena *arena);
stmt_ty Break(int lineno, PyArena *arena);
stmt_ty Continue(int lineno, PyArena *arena);
expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena);
expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena
*arena);
expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena);
expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena);
expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena
*arena);
expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena);
expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
*arena);
expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
*arena);
expr_ty Yield(expr_ty value, int lineno, PyArena *arena);
int col_offset, PyArena *arena);
stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int
col_offset, PyArena *arena);
stmt_ty Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
stmt_ty Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
stmt_ty Pass(int lineno, int col_offset, PyArena *arena);
stmt_ty Break(int lineno, int col_offset, PyArena *arena);
stmt_ty Continue(int lineno, int col_offset, PyArena *arena);
expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
PyArena *arena);
expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
col_offset, PyArena *arena);
expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
PyArena *arena);
expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
PyArena *arena);
expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
col_offset, PyArena *arena);
expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset,
PyArena *arena);
expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
expr_ty Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
starargs, expr_ty kwargs, int lineno, PyArena *arena);
expr_ty Repr(expr_ty value, int lineno, PyArena *arena);
expr_ty Num(object n, int lineno, PyArena *arena);
expr_ty Str(string s, int lineno, PyArena *arena);
starargs, expr_ty kwargs, int lineno, int col_offset, PyArena
*arena);
expr_ty Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
expr_ty Num(object n, int lineno, int col_offset, PyArena *arena);
expr_ty Str(string s, int lineno, int col_offset, PyArena *arena);
expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
lineno, PyArena *arena);
expr_ty Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena);
expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
expr_ty Name(identifier id, expr_context_ty ctx, int lineno, int col_offset,
PyArena *arena);
expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
PyArena *arena);
expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
PyArena *arena);
slice_ty Ellipsis(PyArena *arena);
slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
slice_ty ExtSlice(asdl_seq * dims, PyArena *arena);

3
Include/node.h
View File

@ -11,13 +11,14 @@ typedef struct _node {
short n_type;
char *n_str;
int n_lineno;
int n_col_offset;
int n_nchildren;
struct _node *n_child;
} node;
PyAPI_FUNC(node *) PyNode_New(int type);
PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
char *str, int lineno);
char *str, int lineno, int col_offset);
PyAPI_FUNC(void) PyNode_Free(node *n);
/* Node access functions */

109
Lib/test/test_ast.py
View File

@ -1,4 +1,5 @@
import sys, itertools
import _ast
def to_tuple(t):
if t is None or isinstance(t, (basestring, int, long, complex)):
@ -6,6 +7,8 @@ def to_tuple(t):
elif isinstance(t, list):
return [to_tuple(e) for e in t]
result = [t.__class__.__name__]
if hasattr(t, 'lineno') and hasattr(t, 'col_offset'):
result.append((t.lineno, t.col_offset))
if t._fields is None:
return tuple(result)
for f in t._fields:
@ -106,7 +109,10 @@ eval_tests = [
# List
"[1,2,3]",
# Tuple
"1,2,3"
"1,2,3",
# Combination
"a.b.c.d(a.b[1:2])",
]
# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
@ -121,58 +127,77 @@ if __name__=='__main__' and sys.argv[1:] == ['-g']:
print "run_tests()"
raise SystemExit
def test_order(ast_node, parent_pos):
if not isinstance(ast_node, _ast.AST) or ast_node._fields == None:
return
if isinstance(ast_node, (_ast.expr, _ast.stmt)):
node_pos = (ast_node.lineno, ast_node.col_offset)
assert node_pos >= parent_pos, (node_pos, parent_pos)
parent_pos = (ast_node.lineno, ast_node.col_offset)
for name in ast_node._fields:
value = getattr(ast_node, name)
if isinstance(value, list):
for child in value:
test_order(child, parent_pos)
elif value != None:
test_order(value, parent_pos)
def run_tests():
for input, output, kind in ((exec_tests, exec_results, "exec"),
(single_tests, single_results, "single"),
(eval_tests, eval_results, "eval")):
for i, o in itertools.izip(input, output):
assert to_tuple(compile(i, "?", kind, 0x400)) == o
ast_tree = compile(i, "?", kind, 0x400)
assert to_tuple(ast_tree) == o
test_order(ast_tree, (0, 0))
#### EVERYTHING BELOW IS GENERATED #####
exec_results = [
('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Pass',)], [])]),
('Module', [('ClassDef', 'C', [], [('Pass',)])]),
('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Return', ('Num', 1))], [])]),
('Module', [('Delete', [('Name', 'v', ('Del',))])]),
('Module', [('Assign', [('Name', 'v', ('Store',))], ('Num', 1))]),
('Module', [('AugAssign', ('Name', 'v', ('Load',)), ('Add',), ('Num', 1))]),
('Module', [('Print', ('Name', 'f', ('Load',)), [('Num', 1)], False)]),
('Module', [('For', ('Name', 'v', ('Store',)), ('Name', 'v', ('Load',)), [('Pass',)], [])]),
('Module', [('While', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
('Module', [('If', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
('Module', [('Raise', ('Name', 'Exception', ('Load',)), ('Str', 'string'), None)]),
('Module', [('TryExcept', [('Pass',)], [('excepthandler', ('Name', 'Exception', ('Load',)), None, [('Pass',)])], [])]),
('Module', [('TryFinally', [('Pass',)], [('Pass',)])]),
('Module', [('Assert', ('Name', 'v', ('Load',)), None)]),
('Module', [('Import', [('alias', 'sys', None)])]),
('Module', [('ImportFrom', 'sys', [('alias', 'v', None)], 0)]),
('Module', [('Exec', ('Str', 'v'), None, None)]),
('Module', [('Global', ['v'])]),
('Module', [('Expr', ('Num', 1))]),
('Module', [('Pass',)]),
('Module', [('Break',)]),
('Module', [('Continue',)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Load',)), ('Add',), ('Num', (1, 5), 1))]),
('Module', [('Print', (1, 0), ('Name', (1, 8), 'f', ('Load',)), [('Num', (1, 11), 1)], False)]),
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
('Module', [('Raise', (1, 0), ('Name', (1, 6), 'Exception', ('Load',)), ('Str', (1, 17), 'string'), None)]),
('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('excepthandler', ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
('Module', [('Exec', (1, 0), ('Str', (1, 5), 'v'), None, None)]),
('Module', [('Global', (1, 0), ['v'])]),
('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
('Module', [('Pass', (1, 0))]),
('Module', [('Break', (1, 0))]),
('Module', [('Continue', (1, 0))]),
]
single_results = [
('Interactive', [('Expr', ('BinOp', ('Num', 1), ('Add',), ('Num', 2)))]),
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
]
eval_results = [
('Expression', ('BoolOp', ('And',), [('Name', 'a', ('Load',)), ('Name', 'b', ('Load',))])),
('Expression', ('BinOp', ('Name', 'a', ('Load',)), ('Add',), ('Name', 'b', ('Load',)))),
('Expression', ('UnaryOp', ('Not',), ('Name', 'v', ('Load',)))),
('Expression', ('Lambda', ('arguments', [], None, None, []), ('Name', 'None', ('Load',)))),
('Expression', ('Dict', [('Num', 1)], [('Num', 2)])),
('Expression', ('ListComp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
('Expression', ('GeneratorExp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
('Expression', ('Compare', ('Num', 1), [('Lt',), ('Lt',)], [('Num', 2), ('Num', 3)])),
('Expression', ('Call', ('Name', 'f', ('Load',)), [('Num', 1), ('Num', 2)], [('keyword', 'c', ('Num', 3))], ('Name', 'd', ('Load',)), ('Name', 'e', ('Load',)))),
('Expression', ('Repr', ('Name', 'v', ('Load',)))),
('Expression', ('Num', 10L)),
('Expression', ('Str', 'string')),
('Expression', ('Attribute', ('Name', 'a', ('Load',)), 'b', ('Load',))),
('Expression', ('Subscript', ('Name', 'a', ('Load',)), ('Slice', ('Name', 'b', ('Load',)), ('Name', 'c', ('Load',)), None), ('Load',))),
('Expression', ('Name', 'v', ('Load',))),
('Expression', ('List', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
('Expression', ('Tuple', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
('Expression', ('Repr', (1, 0), ('Name', (1, 1), 'v', ('Load',)))),
('Expression', ('Num', (1, 0), 10L)),
('Expression', ('Str', (1, 0), 'string')),
('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
('Expression', ('Name', (1, 0), 'v', ('Load',))),
('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
]
run_tests()

2
Modules/parsermodule.c
View File

@ -715,7 +715,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
Py_XDECREF(elem);
return (0);
}
err = PyNode_AddChild(root, type, strn, *line_num);
err = PyNode_AddChild(root, type, strn, *line_num, 0);
if (err == E_NOMEM) {
PyMem_DEL(strn);
return (node *) PyErr_NoMemory();

6
Parser/Python.asdl
View File

@ -46,7 +46,8 @@ module Python version "$Revision$"
| Pass | Break | Continue
-- XXX Jython will be different
attributes (int lineno)
-- col_offset is the byte offset in the utf8 string the parser uses
attributes (int lineno, int col_offset)
-- BoolOp() can use left & right?
expr = BoolOp(boolop op, expr* values)
@ -76,7 +77,8 @@ module Python version "$Revision$"
| List(expr* elts, expr_context ctx)
| Tuple(expr *elts, expr_context ctx)
attributes (int lineno)
-- col_offset is the byte offset in the utf8 string the parser uses
attributes (int lineno, int col_offset)
expr_context = Load | Store | Del | AugLoad | AugStore | Param

2
Parser/asdl.py
View File

@ -156,6 +156,8 @@ class ASDLParser(spark.GenericParser, object):
if id.value != "attributes":
raise ASDLSyntaxError(id.lineno,
msg="expected attributes, found %s" % id)
if attributes:
attributes.reverse()
return Sum(sum, attributes)
def p_product(self, (_0, fields, _1)):

3
Parser/node.c
View File

@ -76,7 +76,7 @@ fancy_roundup(int n)
int
PyNode_AddChild(register node *n1, int type, char *str, int lineno)
PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset)
{
const int nch = n1->n_nchildren;
int current_capacity;
@ -103,6 +103,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno)
n->n_type = type;
n->n_str = str;
n->n_lineno = lineno;
n->n_col_offset = col_offset;
n->n_nchildren = 0;
n->n_child = NULL;
return 0;

14
Parser/parser.c
View File

@ -105,11 +105,11 @@ PyParser_Delete(parser_state *ps)
/* PARSER STACK OPERATIONS */
static int
shift(register stack *s, int type, char *str, int newstate, int lineno)
shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset)
{
int err;
assert(!s_empty(s));
err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno);
err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
@ -117,13 +117,13 @@ shift(register stack *s, int type, char *str, int newstate, int lineno)
}
static int
push(register stack *s, int type, dfa *d, int newstate, int lineno)
push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
{
int err;
register node *n;
n = s->s_top->s_parent;
assert(!s_empty(s));
err = PyNode_AddChild(n, type, (char *)NULL, lineno);
err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
@ -213,7 +213,7 @@ future_hack(parser_state *ps)
int
PyParser_AddToken(register parser_state *ps, register int type, char *str,
int lineno, int *expected_ret)
int lineno, int col_offset, int *expected_ret)
{
register int ilabel;
int err;
@ -245,7 +245,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
dfa *d1 = PyGrammar_FindDFA(
ps->p_grammar, nt);
if ((err = push(&ps->p_stack, nt, d1,
arrow, lineno)) > 0) {
arrow, lineno, col_offset)) > 0) {
D(printf(" MemError: push\n"));
return err;
}
@ -255,7 +255,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
/* Shift the token */
if ((err = shift(&ps->p_stack, type, str,
x, lineno)) > 0) {
x, lineno, col_offset)) > 0) {
D(printf(" MemError: shift.\n"));
return err;
}

2
Parser/parser.h
View File

@ -32,7 +32,7 @@ typedef struct {
parser_state *PyParser_New(grammar *g, int start);
void PyParser_Delete(parser_state *ps);
int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno,
int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
int *expected_ret);
void PyGrammar_AddAccelerators(grammar *g);

9
Parser/parsetok.c
View File

@ -130,6 +130,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
int type;
size_t len;
char *str;
int col_offset;
type = PyTokenizer_Get(tok, &a, &b);
if (type == ERRORTOKEN) {
@ -185,9 +186,13 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
len == 4 && str[0] == 'w' && strcmp(str, "with") == 0)
handling_with = 1;
#endif
if (a >= tok->line_start)
col_offset = a - tok->line_start;
else
col_offset = -1;
if ((err_ret->error =
PyParser_AddToken(ps, (int)type, str, tok->lineno,
PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
&(err_ret->expected))) != E_OK) {
if (err_ret->error != E_DONE)
PyObject_FREE(str);

5
Parser/tokenizer.c
View File

@ -764,6 +764,7 @@ tok_nextc(register struct tok_state *tok)
}
if (tok->start == NULL)
tok->buf = tok->cur;
tok->line_start = tok->cur;
tok->lineno++;
tok->inp = end;
return Py_CHARMASK(*tok->cur++);
@ -798,6 +799,7 @@ tok_nextc(register struct tok_state *tok)
}
tok->buf = buf;
tok->cur = tok->buf + oldlen;
tok->line_start = tok->cur;
strcpy(tok->buf + oldlen, new);
PyMem_FREE(new);
tok->inp = tok->buf + newlen;
@ -809,7 +811,9 @@ tok_nextc(register struct tok_state *tok)
if (tok->buf != NULL)
PyMem_DEL(tok->buf);
tok->buf = new;
tok->line_start = tok->buf;
tok->cur = tok->buf;
tok->line_start = tok->buf;
tok->inp = strchr(tok->buf, '\0');
tok->end = tok->inp + 1;
}
@ -877,6 +881,7 @@ tok_nextc(register struct tok_state *tok)
done = tok->inp[-1] == '\n';
}
tok->cur = tok->buf + cur;
tok->line_start = tok->cur;
/* replace "\r\n" with "\n" */
/* For Mac we leave the \r, giving a syntax error */
pt = tok->inp - 2;

1
Parser/tokenizer.h
View File

@ -45,6 +45,7 @@ struct tok_state {
int read_coding_spec; /* whether 'coding:...' has been read */
char *encoding;
int cont_line; /* whether we are in a continuation line. */
const char* line_start; /* pointer to start of current line */
#ifndef PGEN
PyObject *decoding_readline; /* codecs.open(...).readline */
PyObject *decoding_buffer;

169
Python/Python-ast.c
View File

@ -25,6 +25,7 @@ static char *Suite_fields[]={
static PyTypeObject *stmt_type;
static char *stmt_attributes[] = {
"lineno",
"col_offset",
};
static PyObject* ast2obj_stmt(void*);
static PyTypeObject *FunctionDef_type;
@ -142,6 +143,7 @@ static PyTypeObject *Continue_type;
static PyTypeObject *expr_type;
static char *expr_attributes[] = {
"lineno",
"col_offset",
};
static PyObject* ast2obj_expr(void*);
static PyTypeObject *BoolOp_type;
@ -450,7 +452,7 @@ static int init_types(void)
if (!Suite_type) return 0;
stmt_type = make_type("stmt", AST_type, NULL, 0);
if (!stmt_type) return 0;
if (!add_attributes(stmt_type, stmt_attributes, 1)) return 0;
if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
FunctionDef_type = make_type("FunctionDef", stmt_type,
FunctionDef_fields, 4);
if (!FunctionDef_type) return 0;
@ -502,7 +504,7 @@ static int init_types(void)
if (!Continue_type) return 0;
expr_type = make_type("expr", AST_type, NULL, 0);
if (!expr_type) return 0;
if (!add_attributes(expr_type, expr_attributes, 1)) return 0;
if (!add_attributes(expr_type, expr_attributes, 2)) return 0;
BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2);
if (!BoolOp_type) return 0;
BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3);
@ -783,7 +785,7 @@ Suite(asdl_seq * body, PyArena *arena)
stmt_ty
FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
decorators, int lineno, PyArena *arena)
decorators, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
@ -807,12 +809,13 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
p->v.FunctionDef.body = body;
p->v.FunctionDef.decorators = decorators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno,
PyArena *arena)
ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno, int
col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
@ -830,11 +833,12 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno,
p->v.ClassDef.bases = bases;
p->v.ClassDef.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Return(expr_ty value, int lineno, PyArena *arena)
Return(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -845,11 +849,12 @@ Return(expr_ty value, int lineno, PyArena *arena)
p->kind = Return_kind;
p->v.Return.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Delete(asdl_seq * targets, int lineno, PyArena *arena)
Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -860,11 +865,13 @@ Delete(asdl_seq * targets, int lineno, PyArena *arena)
p->kind = Delete_kind;
p->v.Delete.targets = targets;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena)
Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena
*arena)
{
stmt_ty p;
if (!value) {
@ -881,12 +888,13 @@ Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena)
p->v.Assign.targets = targets;
p->v.Assign.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, PyArena
*arena)
AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int
col_offset, PyArena *arena)
{
stmt_ty p;
if (!target) {
@ -914,11 +922,13 @@ AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, PyArena
p->v.AugAssign.op = op;
p->v.AugAssign.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena *arena)
Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int col_offset,
PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -931,12 +941,13 @@ Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena *arena)
p->v.Print.values = values;
p->v.Print.nl = nl;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
lineno, PyArena *arena)
lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!target) {
@ -960,12 +971,13 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
p->v.For.body = body;
p->v.For.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena
*arena)
While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
@ -983,11 +995,13 @@ While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena
p->v.While.body = body;
p->v.While.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena *arena)
If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
@ -1005,12 +1019,13 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena *arena)
p->v.If.body = body;
p->v.If.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno,
PyArena *arena)
int col_offset, PyArena *arena)
{
stmt_ty p;
if (!context_expr) {
@ -1028,11 +1043,13 @@ With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno,
p->v.With.optional_vars = optional_vars;
p->v.With.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena *arena)
Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int col_offset,
PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1045,12 +1062,13 @@ Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena *arena)
p->v.Raise.inst = inst;
p->v.Raise.tback = tback;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno,
PyArena *arena)
int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1063,11 +1081,13 @@ TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno,
p->v.TryExcept.handlers = handlers;
p->v.TryExcept.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena *arena)
TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int col_offset,
PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1079,11 +1099,12 @@ TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena *arena)
p->v.TryFinally.body = body;
p->v.TryFinally.finalbody = finalbody;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena)
Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
@ -1100,11 +1121,12 @@ Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena)
p->v.Assert.test = test;
p->v.Assert.msg = msg;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Import(asdl_seq * names, int lineno, PyArena *arena)
Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1115,12 +1137,13 @@ Import(asdl_seq * names, int lineno, PyArena *arena)
p->kind = Import_kind;
p->v.Import.names = names;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
ImportFrom(identifier module, asdl_seq * names, int level, int lineno, PyArena
*arena)
ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int
col_offset, PyArena *arena)
{
stmt_ty p;
if (!module) {
@ -1138,11 +1161,13 @@ ImportFrom(identifier module, asdl_seq * names, int level, int lineno, PyArena
p->v.ImportFrom.names = names;
p->v.ImportFrom.level = level;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena *arena)
Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int col_offset,
PyArena *arena)
{
stmt_ty p;
if (!body) {
@ -1160,11 +1185,12 @@ Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena *arena)
p->v.Exec.globals = globals;
p->v.Exec.locals = locals;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Global(asdl_seq * names, int lineno, PyArena *arena)
Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1175,11 +1201,12 @@ Global(asdl_seq * names, int lineno, PyArena *arena)
p->kind = Global_kind;
p->v.Global.names = names;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Expr(expr_ty value, int lineno, PyArena *arena)
Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!value) {
@ -1195,11 +1222,12 @@ Expr(expr_ty value, int lineno, PyArena *arena)
p->kind = Expr_kind;
p->v.Expr.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Pass(int lineno, PyArena *arena)
Pass(int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1209,11 +1237,12 @@ Pass(int lineno, PyArena *arena)
}
p->kind = Pass_kind;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Break(int lineno, PyArena *arena)
Break(int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1223,11 +1252,12 @@ Break(int lineno, PyArena *arena)
}
p->kind = Break_kind;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Continue(int lineno, PyArena *arena)
Continue(int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1237,11 +1267,13 @@ Continue(int lineno, PyArena *arena)
}
p->kind = Continue_kind;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena)
BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!op) {
@ -1258,11 +1290,13 @@ BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena)
p->v.BoolOp.op = op;
p->v.BoolOp.values = values;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena *arena)
BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset,
PyArena *arena)
{
expr_ty p;
if (!left) {
@ -1290,11 +1324,13 @@ BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena *arena)
p->v.BinOp.op = op;
p->v.BinOp.right = right;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena)
UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!op) {
@ -1316,11 +1352,13 @@ UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena)
p->v.UnaryOp.op = op;
p->v.UnaryOp.operand = operand;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena)
Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!args) {
@ -1342,11 +1380,13 @@ Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena)
p->v.Lambda.args = args;
p->v.Lambda.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena *arena)
IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset,
PyArena *arena)
{
expr_ty p;
if (!test) {
@ -1374,11 +1414,13 @@ IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena *arena)
p->v.IfExp.body = body;
p->v.IfExp.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena)
Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1390,11 +1432,13 @@ Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena)
p->v.Dict.keys = keys;
p->v.Dict.values = values;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
PyArena *arena)
{
expr_ty p;
if (!elt) {
@ -1411,11 +1455,13 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
p->v.ListComp.elt = elt;
p->v.ListComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
PyArena *arena)
{
expr_ty p;
if (!elt) {
@ -1432,11 +1478,12 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
p->v.GeneratorExp.elt = elt;
p->v.GeneratorExp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Yield(expr_ty value, int lineno, PyArena *arena)
Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1447,12 +1494,13 @@ Yield(expr_ty value, int lineno, PyArena *arena)
p->kind = Yield_kind;
p->v.Yield.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno,
PyArena *arena)
Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno, int
col_offset, PyArena *arena)
{
expr_ty p;
if (!left) {
@ -1470,12 +1518,13 @@ Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno,
p->v.Compare.ops = ops;
p->v.Compare.comparators = comparators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs,
expr_ty kwargs, int lineno, PyArena *arena)
expr_ty kwargs, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!func) {
@ -1495,11 +1544,12 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs,
p->v.Call.starargs = starargs;
p->v.Call.kwargs = kwargs;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Repr(expr_ty value, int lineno, PyArena *arena)
Repr(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@ -1515,11 +1565,12 @@ Repr(expr_ty value, int lineno, PyArena *arena)
p->kind = Repr_kind;
p->v.Repr.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Num(object n, int lineno, PyArena *arena)
Num(object n, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!n) {
@ -1535,11 +1586,12 @@ Num(object n, int lineno, PyArena *arena)
p->kind = Num_kind;
p->v.Num.n = n;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Str(string s, int lineno, PyArena *arena)
Str(string s, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!s) {
@ -1555,12 +1607,13 @@ Str(string s, int lineno, PyArena *arena)
p->kind = Str_kind;
p->v.Str.s = s;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno,
PyArena *arena)
Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@ -1588,12 +1641,13 @@ Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno,
p->v.Attribute.attr = attr;
p->v.Attribute.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno,
PyArena *arena)
Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int
col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@ -1621,11 +1675,13 @@ Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno,
p->v.Subscript.slice = slice;
p->v.Subscript.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena)
Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!id) {
@ -1647,11 +1703,13 @@ Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena)
p->v.Name.id = id;
p->v.Name.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!ctx) {
@ -1668,11 +1726,13 @@ List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
p->v.List.elts = elts;
p->v.List.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!ctx) {
@ -1689,6 +1749,7 @@ Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
p->v.Tuple.elts = elts;
p->v.Tuple.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
@ -2264,6 +2325,9 @@ ast2obj_stmt(void* _o)
value = ast2obj_int(o->lineno);
if (!value) goto failed;
PyObject_SetAttrString(result, "lineno", value);
value = ast2obj_int(o->col_offset);
if (!value) goto failed;
PyObject_SetAttrString(result, "col_offset", value);
return result;
failed:
Py_XDECREF(value);
@ -2580,6 +2644,9 @@ ast2obj_expr(void* _o)
value = ast2obj_int(o->lineno);
if (!value) goto failed;
PyObject_SetAttrString(result, "lineno", value);
value = ast2obj_int(o->col_offset);
if (!value) goto failed;
PyObject_SetAttrString(result, "col_offset", value);
return result;
failed:
Py_XDECREF(value);

172
Python/ast.c
View File

@ -243,7 +243,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
stmts = asdl_seq_new(1, arena);
if (!stmts)
goto error;
asdl_seq_SET(stmts, 0, Pass(n->n_lineno, arena));
asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset, arena));
return Interactive(stmts, arena);
}
else {
@ -564,7 +564,7 @@ compiler_complex_args(struct compiling *c, const node *n)
ast_error(child, "assignment to None");
return NULL;
}
arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child),
arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child), child->n_col_offset,
c->c_arena);
}
else {
@ -573,7 +573,7 @@ compiler_complex_args(struct compiling *c, const node *n)
asdl_seq_SET(args, i, arg);
}
result = Tuple(args, Store, LINENO(n), c->c_arena);
result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena);
if (!set_context(result, Store, n))
return NULL;
return result;
@ -651,7 +651,7 @@ ast_for_arguments(struct compiling *c, const node *n)
goto error;
}
name = Name(NEW_IDENTIFIER(CHILD(ch, 0)),
Param, LINENO(ch), c->c_arena);
Param, LINENO(ch), ch->n_col_offset, c->c_arena);
if (!name)
goto error;
asdl_seq_SET(args, k++, name);
@ -696,14 +696,18 @@ ast_for_dotted_name(struct compiling *c, const node *n)
{
expr_ty e;
identifier id;
int lineno, col_offset;
int i;
REQ(n, dotted_name);
lineno = LINENO(n);
col_offset = n->n_col_offset;
id = NEW_IDENTIFIER(CHILD(n, 0));
if (!id)
return NULL;
e = Name(id, Load, LINENO(n), c->c_arena);
e = Name(id, Load, lineno, col_offset, c->c_arena);
if (!e)
return NULL;
@ -711,7 +715,7 @@ ast_for_dotted_name(struct compiling *c, const node *n)
id = NEW_IDENTIFIER(CHILD(n, i));
if (!id)
return NULL;
e = Attribute(e, id, Load, LINENO(CHILD(n, i)), c->c_arena);
e = Attribute(e, id, Load, lineno, col_offset, c->c_arena);
if (!e)
return NULL;
}
@ -739,7 +743,7 @@ ast_for_decorator(struct compiling *c, const node *n)
name_expr = NULL;
}
else if (NCH(n) == 5) { /* Call with no arguments */
d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n), c->c_arena);
d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
if (!d)
return NULL;
name_expr = NULL;
@ -811,7 +815,7 @@ ast_for_funcdef(struct compiling *c, const node *n)
if (!body)
return NULL;
return FunctionDef(name, args, body, decorator_seq, LINENO(n), c->c_arena);
return FunctionDef(name, args, body, decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
@ -838,7 +842,7 @@ ast_for_lambdef(struct compiling *c, const node *n)
return NULL;
}
return Lambda(args, expression, LINENO(n), c->c_arena);
return Lambda(args, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
@ -857,7 +861,7 @@ ast_for_ifexpr(struct compiling *c, const node *n)
orelse = ast_for_expr(c, CHILD(n, 4));
if (!orelse)
return NULL;
return IfExp(expression, body, orelse, LINENO(n), c->c_arena);
return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset, c->c_arena);
}
/* Count the number of 'for' loop in a list comprehension.
@ -968,7 +972,7 @@ ast_for_listcomp(struct compiling *c, const node *n)
lc = comprehension(asdl_seq_GET(t, 0), expression, NULL,
c->c_arena);
else
lc = comprehension(Tuple(t, Store, LINENO(ch), c->c_arena),
lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, c->c_arena),
expression, NULL, c->c_arena);
if (!lc)
return NULL;
@ -1003,7 +1007,7 @@ ast_for_listcomp(struct compiling *c, const node *n)
asdl_seq_SET(listcomps, i, lc);
}
return ListComp(elt, listcomps, LINENO(n), c->c_arena);
return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
}
/*
@ -1113,7 +1117,7 @@ ast_for_genexp(struct compiling *c, const node *n)
ge = comprehension(asdl_seq_GET(t, 0), expression,
NULL, c->c_arena);
else
ge = comprehension(Tuple(t, Store, LINENO(ch), c->c_arena),
ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, c->c_arena),
expression, NULL, c->c_arena);
if (!ge)
@ -1152,7 +1156,7 @@ ast_for_genexp(struct compiling *c, const node *n)
asdl_seq_SET(genexps, i, ge);
}
return GeneratorExp(elt, genexps, LINENO(n), c->c_arena);
return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
@ -1167,14 +1171,14 @@ ast_for_atom(struct compiling *c, const node *n)
case NAME:
/* All names start in Load context, but may later be
changed. */
return Name(NEW_IDENTIFIER(ch), Load, LINENO(n), c->c_arena);
return Name(NEW_IDENTIFIER(ch), Load, LINENO(n), n->n_col_offset, c->c_arena);
case STRING: {
PyObject *str = parsestrplus(c, n);
if (!str)
return NULL;
PyArena_AddPyObject(c->c_arena, str);
return Str(str, LINENO(n), c->c_arena);
return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
}
case NUMBER: {
PyObject *pynum = parsenumber(STR(ch));
@ -1182,13 +1186,13 @@ ast_for_atom(struct compiling *c, const node *n)
return NULL;
PyArena_AddPyObject(c->c_arena, pynum);
return Num(pynum, LINENO(n), c->c_arena);
return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
}
case LPAR: /* some parenthesized expressions */
ch = CHILD(n, 1);
if (TYPE(ch) == RPAR)
return Tuple(NULL, Load, LINENO(n), c->c_arena);
return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch);
@ -1201,7 +1205,7 @@ ast_for_atom(struct compiling *c, const node *n)
ch = CHILD(n, 1);
if (TYPE(ch) == RSQB)
return List(NULL, Load, LINENO(n), c->c_arena);
return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
REQ(ch, listmaker);
if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
@ -1209,7 +1213,7 @@ ast_for_atom(struct compiling *c, const node *n)
if (!elts)
return NULL;
return List(elts, Load, LINENO(n), c->c_arena);
return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
else
return ast_for_listcomp(c, ch);
@ -1243,14 +1247,14 @@ ast_for_atom(struct compiling *c, const node *n)
asdl_seq_SET(values, i / 4, expression);
}
return Dict(keys, values, LINENO(n), c->c_arena);
return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
}
case BACKQUOTE: { /* repr */
expr_ty expression = ast_for_testlist(c, CHILD(n, 1));
if (!expression)
return NULL;
return Repr(expression, LINENO(n), c->c_arena);
return Repr(expression, LINENO(n), n->n_col_offset, c->c_arena);
}
default:
PyErr_Format(PyExc_SystemError, "unhandled atom %d", TYPE(ch));
@ -1353,7 +1357,7 @@ ast_for_binop(struct compiling *c, const node *n)
if (!operator)
return NULL;
result = BinOp(expr1, operator, expr2, LINENO(n), c->c_arena);
result = BinOp(expr1, operator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
if (!result)
return NULL;
@ -1371,7 +1375,7 @@ ast_for_binop(struct compiling *c, const node *n)
return NULL;
tmp_result = BinOp(result, operator, tmp,
LINENO(next_oper), c->c_arena);
LINENO(next_oper), next_oper->n_col_offset, c->c_arena);
if (!tmp)
return NULL;
result = tmp_result;
@ -1389,13 +1393,13 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
REQ(n, trailer);
if (TYPE(CHILD(n, 0)) == LPAR) {
if (NCH(n) == 2)
return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), c->c_arena);
return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
else
return ast_for_call(c, CHILD(n, 1), left_expr);
}
else if (TYPE(CHILD(n, 0)) == DOT ) {
return Attribute(left_expr, NEW_IDENTIFIER(CHILD(n, 1)), Load,
LINENO(n), c->c_arena);
LINENO(n), n->n_col_offset, c->c_arena);
}
else {
REQ(CHILD(n, 0), LSQB);
@ -1405,7 +1409,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
slice_ty slc = ast_for_slice(c, CHILD(n, 0));
if (!slc)
return NULL;
return Subscript(left_expr, slc, Load, LINENO(n), c->c_arena);
return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
else {
/* The grammar is ambiguous here. The ambiguity is resolved
@ -1430,7 +1434,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
}
if (!simple) {
return Subscript(left_expr, ExtSlice(slices, c->c_arena),
Load, LINENO(n), c->c_arena);
Load, LINENO(n), n->n_col_offset, c->c_arena);
}
/* extract Index values and put them in a Tuple */
elts = asdl_seq_new(asdl_seq_LEN(slices), c->c_arena);
@ -1439,11 +1443,11 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
assert(slc->kind == Index_kind && slc->v.Index.value);
asdl_seq_SET(elts, j, slc->v.Index.value);
}
e = Tuple(elts, Load, LINENO(n), c->c_arena);
e = Tuple(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
if (!e)
return NULL;
return Subscript(left_expr, Index(e, c->c_arena),
Load, LINENO(n), c->c_arena);
Load, LINENO(n), n->n_col_offset, c->c_arena);
}
}
}
@ -1468,13 +1472,15 @@ ast_for_power(struct compiling *c, const node *n)
tmp = ast_for_trailer(c, ch, e);
if (!tmp)
return NULL;
tmp->lineno = e->lineno;
tmp->col_offset = e->col_offset;
e = tmp;
}
if (TYPE(CHILD(n, NCH(n) - 1)) == factor) {
expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
if (!f)
return NULL;
tmp = BinOp(e, Pow, f, LINENO(n), c->c_arena);
tmp = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena);
if (!tmp)
return NULL;
e = tmp;
@ -1542,9 +1548,9 @@ ast_for_expr(struct compiling *c, const node *n)
asdl_seq_SET(seq, i / 2, e);
}
if (!strcmp(STR(CHILD(n, 1)), "and"))
return BoolOp(And, seq, LINENO(n), c->c_arena);
return BoolOp(And, seq, LINENO(n), n->n_col_offset, c->c_arena);
assert(!strcmp(STR(CHILD(n, 1)), "or"));
return BoolOp(Or, seq, LINENO(n), c->c_arena);
return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena);
case not_test:
if (NCH(n) == 1) {
n = CHILD(n, 0);
@ -1555,7 +1561,7 @@ ast_for_expr(struct compiling *c, const node *n)
if (!expression)
return NULL;
return UnaryOp(Not, expression, LINENO(n), c->c_arena);
return UnaryOp(Not, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
case comparison:
if (NCH(n) == 1) {
@ -1594,7 +1600,7 @@ ast_for_expr(struct compiling *c, const node *n)
return NULL;
}
return Compare(expression, ops, cmps, LINENO(n), c->c_arena);
return Compare(expression, ops, cmps, LINENO(n), n->n_col_offset, c->c_arena);
}
break;
@ -1620,7 +1626,7 @@ ast_for_expr(struct compiling *c, const node *n)
if (!exp)
return NULL;
}
return Yield(exp, LINENO(n), c->c_arena);
return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena);
}
case factor: {
expr_ty expression;
@ -1636,11 +1642,11 @@ ast_for_expr(struct compiling *c, const node *n)
switch (TYPE(CHILD(n, 0))) {
case PLUS:
return UnaryOp(UAdd, expression, LINENO(n), c->c_arena);
return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset, c->c_arena);
case MINUS:
return UnaryOp(USub, expression, LINENO(n), c->c_arena);
return UnaryOp(USub, expression, LINENO(n), n->n_col_offset, c->c_arena);
case TILDE:
return UnaryOp(Invert, expression, LINENO(n), c->c_arena);
return UnaryOp(Invert, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError, "unhandled factor: %d",
TYPE(CHILD(n, 0)));
@ -1761,7 +1767,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
}
}
return Call(func, args, keywords, vararg, kwarg, LINENO(n), c->c_arena);
return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_offset, c->c_arena);
}
static expr_ty
@ -1787,7 +1793,7 @@ ast_for_testlist(struct compiling *c, const node* n)
asdl_seq *tmp = seq_for_testlist(c, n);
if (!tmp)
return NULL;
return Tuple(tmp, Load, LINENO(n), c->c_arena);
return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
}
@ -1841,7 +1847,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
if (!e)
return NULL;
return Expr(e, LINENO(n), c->c_arena);
return Expr(e, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (TYPE(CHILD(n, 1)) == augassign) {
expr_ty expr1, expr2;
@ -1851,7 +1857,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
if (TYPE(ch) == testlist)
expr1 = ast_for_testlist(c, ch);
else
expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch),
expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch), n->n_col_offset,
c->c_arena);
if (!expr1)
@ -1883,7 +1889,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
if (TYPE(ch) == testlist)
expr2 = ast_for_testlist(c, ch);
else
expr2 = Yield(ast_for_expr(c, ch), LINENO(ch), c->c_arena);
expr2 = Yield(ast_for_expr(c, ch), LINENO(ch), ch->n_col_offset, c->c_arena);
if (!expr2)
return NULL;
@ -1891,7 +1897,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
if (!operator)
return NULL;
return AugAssign(expr1, operator, expr2, LINENO(n), c->c_arena);
return AugAssign(expr1, operator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
}
else {
int i;
@ -1929,7 +1935,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
expression = ast_for_expr(c, value);
if (!expression)
return NULL;
return Assign(targets, expression, LINENO(n), c->c_arena);
return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
}
@ -1961,7 +1967,7 @@ ast_for_print_stmt(struct compiling *c, const node *n)
asdl_seq_SET(seq, j, expression);
}
nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true;
return Print(dest, seq, nl, LINENO(n), c->c_arena);
return Print(dest, seq, nl, LINENO(n), n->n_col_offset, c->c_arena);
}
static asdl_seq *
@ -1998,7 +2004,7 @@ ast_for_del_stmt(struct compiling *c, const node *n)
expr_list = ast_for_exprlist(c, CHILD(n, 1), Del);
if (!expr_list)
return NULL;
return Delete(expr_list, LINENO(n), c->c_arena);
return Delete(expr_list, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
@ -2020,32 +2026,32 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
ch = CHILD(n, 0);
switch (TYPE(ch)) {
case break_stmt:
return Break(LINENO(n), c->c_arena);
return Break(LINENO(n), n->n_col_offset, c->c_arena);
case continue_stmt:
return Continue(LINENO(n), c->c_arena);
return Continue(LINENO(n), n->n_col_offset, c->c_arena);
case yield_stmt: { /* will reduce to yield_expr */
expr_ty exp = ast_for_expr(c, CHILD(ch, 0));
if (!exp)
return NULL;
return Expr(exp, LINENO(n), c->c_arena);
return Expr(exp, LINENO(n), n->n_col_offset, c->c_arena);
}
case return_stmt:
if (NCH(ch) == 1)
return Return(NULL, LINENO(n), c->c_arena);
return Return(NULL, LINENO(n), n->n_col_offset, c->c_arena);
else {
expr_ty expression = ast_for_testlist(c, CHILD(ch, 1));
if (!expression)
return NULL;
return Return(expression, LINENO(n), c->c_arena);
return Return(expression, LINENO(n), n->n_col_offset, c->c_arena);
}
case raise_stmt:
if (NCH(ch) == 1)
return Raise(NULL, NULL, NULL, LINENO(n), c->c_arena);
return Raise(NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
else if (NCH(ch) == 2) {
expr_ty expression = ast_for_expr(c, CHILD(ch, 1));
if (!expression)
return NULL;
return Raise(expression, NULL, NULL, LINENO(n), c->c_arena);
return Raise(expression, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(ch) == 4) {
expr_ty expr1, expr2;
@ -2057,7 +2063,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
if (!expr2)
return NULL;
return Raise(expr1, expr2, NULL, LINENO(n), c->c_arena);
return Raise(expr1, expr2, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(ch) == 6) {
expr_ty expr1, expr2, expr3;
@ -2072,7 +2078,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
if (!expr3)
return NULL;
return Raise(expr1, expr2, expr3, LINENO(n), c->c_arena);
return Raise(expr1, expr2, expr3, LINENO(n), n->n_col_offset, c->c_arena);
}
default:
PyErr_Format(PyExc_SystemError,
@ -2167,10 +2173,14 @@ ast_for_import_stmt(struct compiling *c, const node *n)
import_from: 'from' ('.'* dotted_name | '.') 'import'
('*' | '(' import_as_names ')' | import_as_names)
*/
int lineno;
int col_offset;
int i;
asdl_seq *aliases;
REQ(n, import_stmt);
lineno = LINENO(n);
col_offset = n->n_col_offset;
n = CHILD(n, 0);
if (TYPE(n) == import_name) {
n = CHILD(n, 1);
@ -2184,11 +2194,10 @@ ast_for_import_stmt(struct compiling *c, const node *n)
return NULL;
asdl_seq_SET(aliases, i / 2, import_alias);
}
return Import(aliases, LINENO(n), c->c_arena);
return Import(aliases, lineno, col_offset, c->c_arena);
}
else if (TYPE(n) == import_from) {
int n_children;
int lineno = LINENO(n);
int idx, ndots = 0;
alias_ty mod = NULL;
identifier modname;
@ -2259,7 +2268,7 @@ ast_for_import_stmt(struct compiling *c, const node *n)
modname = mod->name;
else
modname = new_identifier("", c->c_arena);
return ImportFrom(modname, aliases, ndots, lineno,
return ImportFrom(modname, aliases, ndots, lineno, col_offset,
c->c_arena);
}
PyErr_Format(PyExc_SystemError,
@ -2286,7 +2295,7 @@ ast_for_global_stmt(struct compiling *c, const node *n)
return NULL;
asdl_seq_SET(s, i / 2, name);
}
return Global(s, LINENO(n), c->c_arena);
return Global(s, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
@ -2317,7 +2326,7 @@ ast_for_exec_stmt(struct compiling *c, const node *n)
return NULL;
}
return Exec(expr1, globals, locals, LINENO(n), c->c_arena);
return Exec(expr1, globals, locals, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
@ -2329,7 +2338,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
expr_ty expression = ast_for_expr(c, CHILD(n, 1));
if (!expression)
return NULL;
return Assert(expression, NULL, LINENO(n), c->c_arena);
return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(n) == 4) {
expr_ty expr1, expr2;
@ -2341,7 +2350,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
if (!expr2)
return NULL;
return Assert(expr1, expr2, LINENO(n), c->c_arena);
return Assert(expr1, expr2, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
"improper number of parts to 'assert' statement: %d",
@ -2436,7 +2445,7 @@ ast_for_if_stmt(struct compiling *c, const node *n)
if (!suite_seq)
return NULL;
return If(expression, suite_seq, NULL, LINENO(n), c->c_arena);
return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
s = STR(CHILD(n, 4));
@ -2458,7 +2467,7 @@ ast_for_if_stmt(struct compiling *c, const node *n)
if (!seq2)
return NULL;
return If(expression, seq1, seq2, LINENO(n), c->c_arena);
return If(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (s[2] == 'i') {
int i, n_elif, has_else = 0;
@ -2491,7 +2500,7 @@ ast_for_if_stmt(struct compiling *c, const node *n)
return NULL;
asdl_seq_SET(orelse, 0, If(expression, seq1, seq2,
LINENO(CHILD(n, NCH(n) - 6)),
LINENO(CHILD(n, NCH(n) - 6)), CHILD(n, NCH(n) - 6)->n_col_offset,
c->c_arena));
/* the just-created orelse handled the last elif */
n_elif--;
@ -2513,12 +2522,12 @@ ast_for_if_stmt(struct compiling *c, const node *n)
asdl_seq_SET(new, 0,
If(expression, suite_seq, orelse,
LINENO(CHILD(n, off)), c->c_arena));
LINENO(CHILD(n, off)), CHILD(n, off)->n_col_offset, c->c_arena));
orelse = new;
}
return If(ast_for_expr(c, CHILD(n, 1)),
ast_for_suite(c, CHILD(n, 3)),
orelse, LINENO(n), c->c_arena);
orelse, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
@ -2542,7 +2551,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
suite_seq = ast_for_suite(c, CHILD(n, 3));
if (!suite_seq)
return NULL;
return While(expression, suite_seq, NULL, LINENO(n), c->c_arena);
return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(n) == 7) {
expr_ty expression;
@ -2558,7 +2567,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
if (!seq2)
return NULL;
return While(expression, seq1, seq2, LINENO(n), c->c_arena);
return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
@ -2588,7 +2597,7 @@ ast_for_for_stmt(struct compiling *c, const node *n)
if (asdl_seq_LEN(_target) == 1)
target = asdl_seq_GET(_target, 0);
else
target = Tuple(_target, Store, LINENO(n), c->c_arena);
target = Tuple(_target, Store, LINENO(n), n->n_col_offset, c->c_arena);
expression = ast_for_testlist(c, CHILD(n, 3));
if (!expression)
@ -2597,7 +2606,7 @@ ast_for_for_stmt(struct compiling *c, const node *n)
if (!suite_seq)
return NULL;
return For(target, expression, suite_seq, seq, LINENO(n), c->c_arena);
return For(target, expression, suite_seq, seq, LINENO(n), n->n_col_offset, c->c_arena);
}
static excepthandler_ty
@ -2711,7 +2720,7 @@ ast_for_try_stmt(struct compiling *c, const node *n)
asdl_seq_SET(handlers, i, e);
}
except_st = TryExcept(body, handlers, orelse, LINENO(n), c->c_arena);
except_st = TryExcept(body, handlers, orelse, LINENO(n), n->n_col_offset, c->c_arena);
if (!finally)
return except_st;
@ -2725,7 +2734,7 @@ ast_for_try_stmt(struct compiling *c, const node *n)
/* must be a try ... finally (except clauses are in body, if any exist) */
assert(finally != NULL);
return TryFinally(body, finally, LINENO(n), c->c_arena);
return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
@ -2765,7 +2774,8 @@ ast_for_with_stmt(struct compiling *c, const node *n)
if (!suite_seq) {
return NULL;
}
return With(context_expr, optional_vars, suite_seq, LINENO(n), c->c_arena);
return With(context_expr, optional_vars, suite_seq, LINENO(n),
n->n_col_offset, c->c_arena);
}
static stmt_ty
@ -2785,7 +2795,7 @@ ast_for_classdef(struct compiling *c, const node *n)
s = ast_for_suite(c, CHILD(n, 3));
if (!s)
return NULL;
return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n),
return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), n->n_col_offset,
c->c_arena);
}
/* check for empty base list */
@ -2793,7 +2803,7 @@ ast_for_classdef(struct compiling *c, const node *n)
s = ast_for_suite(c, CHILD(n,5));
if (!s)
return NULL;
return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n),
return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), n->n_col_offset,
c->c_arena);
}
@ -2805,7 +2815,7 @@ ast_for_classdef(struct compiling *c, const node *n)
s = ast_for_suite(c, CHILD(n, 6));
if (!s)
return NULL;
return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n),
return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n), n->n_col_offset,
c->c_arena);
}
@ -2835,7 +2845,7 @@ ast_for_stmt(struct compiling *c, const node *n)
case del_stmt:
return ast_for_del_stmt(c, n);
case pass_stmt:
return Pass(LINENO(n), c->c_arena);
return Pass(LINENO(n), n->n_col_offset, c->c_arena);
case flow_stmt:
return ast_for_flow_stmt(c, n);
case import_stmt:

4
Python/compile.c
View File

@ -3635,7 +3635,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
switch (e->kind) {
case Attribute_kind:
auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
AugLoad, e->lineno, c->c_arena);
AugLoad, e->lineno, e->col_offset, c->c_arena);
if (auge == NULL)
return 0;
VISIT(c, expr, auge);
@ -3646,7 +3646,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
break;
case Subscript_kind:
auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
AugLoad, e->lineno, c->c_arena);
AugLoad, e->lineno, e->col_offset, c->c_arena);
if (auge == NULL)
return 0;
VISIT(c, expr, auge);