Skip to content

bpo-46042: Improve SyntaxError locations in the symbol table #30059

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pablogsal merged 3 commits into from
Dec 11, 2021
Merged

bpo-46042: Improve SyntaxError locations in the symbol table #30059

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fa8df44
bpo-46042: Improve SyntaxError locations in the symbol table
pablogsal Dec 11, 2021
6d7034e
Fix error for import *
pablogsal Dec 11, 2021
f9dcc8b
Correctly point to yield inside comprehensions
pablogsal Dec 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Correctly point to yield inside comprehensions
  • Loading branch information
@pablogsal
pablogsal committed Dec 11, 2021
commit f9dcc8b5beacacd4a75e8f1d07a5e42903dd10f7
11 changes: 9 additions & 2 deletions Include/internal/pycore_symtable.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@ struct _mod; // Type defined in pycore_ast.h
typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, AnnotationBlock }
_Py_block_ty;

typedef enum _comprehension_type {
NoComprehension = 0,
ListComprehension = 1,
DictComprehension = 2,
SetComprehension = 3,
GeneratorExpression = 4 } _Py_comprehension_ty;

struct _symtable_entry;

struct symtable {
Expand Down Expand Up @@ -42,14 +49,14 @@ typedef struct _symtable_entry {
PyObject *ste_varnames; /* list of function parameters */
PyObject *ste_children; /* list of child blocks */
PyObject *ste_directives;/* locations of global and nonlocal statements */
_Py_block_ty ste_type; /* module, class, or function */
_Py_block_ty ste_type; /* module, class or function */
int ste_nested; /* true if block is nested */
unsigned ste_free : 1; /* true if block has free variables */
unsigned ste_child_free : 1; /* true if a child block has free vars,
including free refs to globals */
unsigned ste_generator : 1; /* true if namespace is a generator */
unsigned ste_coroutine : 1; /* true if namespace is a coroutine */
unsigned ste_comprehension : 1; /* true if namespace is a list comprehension */
_Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */
unsigned ste_varargs : 1; /* true if block has varargs */
unsigned ste_varkeywords : 1; /* true if block has varkeywords */
unsigned ste_returns_value : 1; /* true if namespace uses return with
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ def baz():
check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1)

# Errors thrown by symtable.c
check('x = [(yield i) for i in range(3)]', 1, 5)
check('x = [(yield i) for i in range(3)]', 1, 7)
check('def f():\n from _ import *', 2, 17)
check('def f(x, x):\n pass', 1, 10)
check('{i for i in range(5) if (j := 0) for j in range(5)}', 1, 38)
Expand Down
53 changes: 36 additions & 17 deletions Python/symtable.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@
#define ST_LOCATION(x) \
(x)->ste_lineno, (x)->ste_col_offset, (x)->ste_end_lineno, (x)->ste_end_col_offset


static PySTEntryObject *
ste_new(struct symtable *st, identifier name, _Py_block_ty block,
void *key, int lineno, int col_offset,
Expand Down Expand Up @@ -103,7 +102,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_child_free = 0;
ste->ste_generator = 0;
ste->ste_coroutine = 0;
ste->ste_comprehension = 0;
ste->ste_comprehension = NoComprehension;
ste->ste_returns_value = 0;
ste->ste_needs_class_closure = 0;
ste->ste_comp_iter_target = 0;
Expand Down Expand Up @@ -228,6 +227,7 @@ static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty);
static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty);


static identifier top = NULL, lambda = NULL, genexpr = NULL,
Expand Down Expand Up @@ -1656,13 +1656,19 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
if (e->v.Yield.value)
VISIT(st, expr, e->v.Yield.value);
st->st_cur->ste_generator = 1;
if (st->st_cur->ste_comprehension) {
return symtable_raise_if_comprehension_block(st, e);
}
break;
case YieldFrom_kind:
if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
VISIT_QUIT(st, 0);
}
VISIT(st, expr, e->v.YieldFrom.value);
st->st_cur->ste_generator = 1;
if (st->st_cur->ste_comprehension) {
return symtable_raise_if_comprehension_block(st, e);
}
break;
case Await_kind:
if (!symtable_raise_if_annotation_block(st, "await expression", e)) {
Expand Down Expand Up @@ -2026,10 +2032,23 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
e->end_lineno, e->end_col_offset)) {
return 0;
}
switch(e->kind) {
case ListComp_kind:
st->st_cur->ste_comprehension = ListComprehension;
break;
case SetComp_kind:
st->st_cur->ste_comprehension = SetComprehension;
break;
case DictComp_kind:
st->st_cur->ste_comprehension = DictComprehension;
break;
default:
st->st_cur->ste_comprehension = GeneratorExpression;
break;
}
if (outermost->is_async) {
st->st_cur->ste_coroutine = 1;
}
st->st_cur->ste_comprehension = 1;

/* Outermost iter is received as an argument */
if (!symtable_implicit_arg(st, 0)) {
Expand All @@ -2046,20 +2065,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
if (value)
VISIT(st, expr, value);
VISIT(st, expr, elt);
if (st->st_cur->ste_generator) {
PyErr_SetString(PyExc_SyntaxError,
(e->kind == ListComp_kind) ? "'yield' inside list comprehension" :
(e->kind == SetComp_kind) ? "'yield' inside set comprehension" :
(e->kind == DictComp_kind) ? "'yield' inside dict comprehension" :
"'yield' inside generator expression");
PyErr_RangedSyntaxLocationObject(st->st_filename,
st->st_cur->ste_lineno,
st->st_cur->ste_col_offset + 1,
st->st_cur->ste_end_lineno,
st->st_cur->ste_end_col_offset + 1);
symtable_exit_block(st);
return 0;
}
st->st_cur->ste_generator = is_generator;
int is_async = st->st_cur->ste_coroutine && !is_generator;
if (!symtable_exit_block(st)) {
Expand Down Expand Up @@ -2120,6 +2125,20 @@ symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_t
return 0;
}

static int
symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) {
_Py_comprehension_ty type = st->st_cur->ste_comprehension;
PyErr_SetString(PyExc_SyntaxError,
(type == ListComprehension) ? "'yield' inside list comprehension" :
(type == SetComprehension) ? "'yield' inside set comprehension" :
(type == DictComprehension) ? "'yield' inside dict comprehension" :
"'yield' inside generator expression");
PyErr_RangedSyntaxLocationObject(st->st_filename,
e->lineno, e->col_offset + 1,
e->end_lineno, e->end_col_offset + 1);
VISIT_QUIT(st, 0);
}

struct symtable *
_Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
int start, PyCompilerFlags *flags)
Expand Down