Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
bpo-43224: Implement PEP 646 grammar changes
  • Loading branch information
mrahtz committed Jan 30, 2022
commit 7ecb7ba6db087e96ef2be43901f43817adb9a8c0
9 changes: 8 additions & 1 deletion Grammar/python.gram
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,8 @@ slash_with_default[SlashWithDefault*]:
star_etc[StarEtc*]:
| '*' a=param_no_default b=param_maybe_default* c=[kwds] {
_PyPegen_star_etc(p, a, b, c) }
| '*' a=param_no_default_star_annotation b=param_maybe_default* c=[kwds] {
_PyPegen_star_etc(p, a, b, c) }
| '*' ',' b=param_maybe_default+ c=[kwds] {
_PyPegen_star_etc(p, NULL, b, c) }
| a=kwds { _PyPegen_star_etc(p, NULL, NULL, a) }
Expand All @@ -331,14 +333,19 @@ kwds[arg_ty]: '**' a=param_no_default { a }
param_no_default[arg_ty]:
| a=param ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
param_no_default_star_annotation[arg_ty]:
Comment thread
pablogsal marked this conversation as resolved.
| a=param_star_annotation ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param_star_annotation tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
param_with_default[NameDefaultPair*]:
| a=param c=default ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
param_maybe_default[NameDefaultPair*]:
| a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
param_star_annotation[arg_ty]: a=NAME b=star_annotation { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
annotation[expr_ty]: ':' a=expression { a }
star_annotation[expr_ty]: ':' a=star_expression { a }
default[expr_ty]: '=' a=expression { a }

# If statement
Expand Down Expand Up @@ -780,7 +787,7 @@ primary[expr_ty]:

slices[expr_ty]:
| a=slice !',' { a }
| a[asdl_expr_seq*]=','.slice+ [','] { _PyAST_Tuple(a, Load, EXTRA) }
| a[asdl_expr_seq*]=','.(slice | starred_expression)+ [','] { _PyAST_Tuple(a, Load, EXTRA) }
Comment thread
mrahtz marked this conversation as resolved.

slice[expr_ty]:
| a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] { _PyAST_Slice(a, b, c, EXTRA) }
Expand Down
25 changes: 24 additions & 1 deletion Lib/test/test_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from test import support

def to_tuple(t):
if t is None or isinstance(t, (str, int, complex)):
if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis:
return t
elif isinstance(t, list):
return [to_tuple(e) for e in t]
Expand Down Expand Up @@ -46,10 +46,20 @@ def to_tuple(t):
"def f(a=0): pass",
# FunctionDef with varargs
"def f(*args): pass",
# FunctionDef with varargs as TypeVarTuple
"def f(*args: *Ts): pass",
# FunctionDef with varargs as unpacked Tuple
"def f(*args: *tuple[int, ...]): pass",
# FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple
"def f(*args: *tuple[int, *Ts]): pass",
# FunctionDef with kwargs
"def f(**kwargs): pass",
# FunctionDef with all kind of args and docstring
"def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'",
# FunctionDef with type annotation on return involving unpacking
"def f() -> tuple[*Ts]: pass",
"def f() -> tuple[int, *Ts]: pass",
"def f() -> tuple[int, *tuple[int, ...]]: pass",
# ClassDef
"class C:pass",
# ClassDef with docstring
Expand All @@ -65,6 +75,10 @@ def to_tuple(t):
"a,b = c",
"(a,b) = c",
"[a,b] = c",
# AnnAssign with unpacked types
"x: tuple[*Ts]",
"x: tuple[int, *Ts]",
"x: tuple[int, *tuple[str, ...]]",
# AugAssign
"v += 1",
# For
Expand Down Expand Up @@ -2316,8 +2330,14 @@ def main():
('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [])], []),
Expand All @@ -2327,6 +2347,9 @@ def main():
('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []),
('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []),
('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []),
Expand Down
268 changes: 268 additions & 0 deletions Lib/test/test_pep646_syntax.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
doctests = """

Setup

>>> class AClass:
... def __init__(self):
... self._setitem_name = None
... self._setitem_val = None
... def __setitem__(self, name, val):
... self._setitem_name = name
... self._setitem_val = val
... def __repr__(self):
... return f"A[{self._setitem_name}]={self._setitem_val}"
... def __getitem__(self, name):
... return ParameterisedA(name)
>>> class ParameterisedA:
... def __init__(self, name):
... self._name = name
... def __repr__(self):
... return f"A[{self._name}]"
... def __iter__(self):
... for p in self._name:
... yield p
>>> class B:
... def __iter__(self):
... yield StarredB()
... def __repr__(self):
... return "B"
>>> class StarredB:
... def __repr__(self):
... return "StarredB"
>>> A = AClass()
>>> b = B()

Slices that are supposed to work, starring our custom B class

>>> A[*b]
A[(StarredB,)]
>>> A[*b, *b]
A[(StarredB, StarredB)]
>>> A[b, *b]
A[(B, StarredB)]
>>> A[*b, b]
A[(StarredB, B)]
>>> A[b, b, *b]
A[(B, B, StarredB)]
>>> A[*b, b, b]
A[(StarredB, B, B)]
>>> A[b, *b, b]
A[(B, StarredB, B)]
>>> A[b, b, *b, b]
A[(B, B, StarredB, B)]
>>> A[b, *b, b, b]
A[(B, StarredB, B, B)]
>>> A[A[b, *b, b]]
A[A[(B, StarredB, B)]]
>>> A[*A[b, *b, b]]
A[(B, StarredB, B)]
>>> A[b, ...]
A[(B, Ellipsis)]
>>> A[*A[b, ...]]
A[(B, Ellipsis)]

Slices assignments that are supposed to work, starring our custom B class
Comment thread
mrahtz marked this conversation as resolved.
Outdated

>>> A[*b] = 1; A
A[(StarredB,)]=1
>>> A[*b, *b] = 1; A
A[(StarredB, StarredB)]=1
>>> A[b, *b] = 1; A
A[(B, StarredB)]=1
>>> A[*b, b] = 1; A
A[(StarredB, B)]=1
>>> A[b, b, *b] = 1; A
A[(B, B, StarredB)]=1
>>> A[*b, b, b] = 1; A
A[(StarredB, B, B)]=1
>>> A[b, *b, b] = 1; A
A[(B, StarredB, B)]=1
>>> A[b, b, *b, b] = 1; A
A[(B, B, StarredB, B)]=1
>>> A[b, *b, b, b] = 1; A
A[(B, StarredB, B, B)]=1
>>> A[A[b, *b, b]] = 1; A
A[A[(B, StarredB, B)]]=1
>>> A[*A[b, *b, b]] = 1; A
A[(B, StarredB, B)]=1
>>> A[b, ...] = 1; A
A[(B, Ellipsis)]=1
>>> A[*A[b, ...]] = 1; A
A[(B, Ellipsis)]=1

Slices that are supposed to work, starring a list

>>> l = [1, 2, 3]
>>> A[*l]
A[(1, 2, 3)]
>>> A[*l, 4]
A[(1, 2, 3, 4)]
>>> A[0, *l]
A[(0, 1, 2, 3)]
>>> A[1:2, *l]
A[(slice(1, 2, None), 1, 2, 3)]
>>> repr(A[1:2, *l]) == repr(A[1:2, 1, 2, 3])
True

Slice assignments that are supposed to work, starring a list

>>> l = [1, 2, 3]
>>> A[*l] = 1; A
A[(1, 2, 3)]=1
>>> A[*l, 4] = 1; A
A[(1, 2, 3, 4)]=1
>>> A[0, *l] = 1; A
A[(0, 1, 2, 3)]=1
>>> A[1:2, *l] = 1; A
A[(slice(1, 2, None), 1, 2, 3)]=1

Slices that are supposed to work, starring a tuple

>>> t = (1, 2, 3)
>>> A[*t]
A[(1, 2, 3)]
>>> A[*t, 4]
A[(1, 2, 3, 4)]
>>> A[0, *t]
A[(0, 1, 2, 3)]
>>> A[1:2, *t]
A[(slice(1, 2, None), 1, 2, 3)]
>>> repr(A[1:2, *t]) == repr(A[1:2, 1, 2, 3])
True

Slice assignments that are supposed to work, starring a tuple

>>> t = (1, 2, 3)
>>> A[*t] = 1; A
A[(1, 2, 3)]=1
>>> A[*t, 4] = 1; A
A[(1, 2, 3, 4)]=1
>>> A[0, *t] = 1; A
A[(0, 1, 2, 3)]=1
>>> A[1:2, *t] = 1; A
A[(slice(1, 2, None), 1, 2, 3)]=1

Starring an expression (rather than a name) in a slice

>>> def returns_list():
... return [1, 2, 3]
>>> A[returns_list()]
A[[1, 2, 3]]
>>> A[returns_list(), 4]
A[([1, 2, 3], 4)]
>>> A[*returns_list()]
A[(1, 2, 3)]
>>> A[*returns_list(), 4]
A[(1, 2, 3, 4)]
>>> A[0, *returns_list()]
A[(0, 1, 2, 3)]
>>> A[*returns_list(), *returns_list()]
A[(1, 2, 3, 1, 2, 3)]

Starring an expression (rather than a name) in a slice assignment

>>> def returns_list():
... return [1, 2, 3]
>>> A[returns_list()] = 1; A
A[[1, 2, 3]]=1
>>> A[returns_list(), 4] = 1; A
A[([1, 2, 3], 4)]=1
>>> A[*returns_list()] = 1; A
A[(1, 2, 3)]=1
>>> A[*returns_list(), 4] = 1; A
A[(1, 2, 3, 4)]=1
>>> A[0, *returns_list()] = 1; A
A[(0, 1, 2, 3)]=1
>>> A[*returns_list(), *returns_list()] = 1; A
A[(1, 2, 3, 1, 2, 3)]=1

Slices that should fail

>>> A[:*b]
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> A[*b:]
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> A[*b:*b]
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> A[**b]
Traceback (most recent call last):
...
SyntaxError: invalid syntax

Slice assignments that should fail

>>> A[:*b] = 1
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> A[*b:] = 1
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> A[*b:*b] = 1
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> A[**b] = 1
Traceback (most recent call last):
...
SyntaxError: invalid syntax

*args annotated as starred expression

>>> def f1(*args: *b): pass
>>> f1.__annotations__
{'args': StarredB}

>>> def f2(*args: *b, arg1): pass
>>> f2.__annotations__
{'args': StarredB}

>>> def f3(*args: *b, arg1: int): pass
>>> f3.__annotations__
{'args': StarredB, 'arg1': <class 'int'>}

>>> def f4(*args: *b, arg1: int = 2): pass
>>> f4.__annotations__
{'args': StarredB, 'arg1': <class 'int'>}

>>> def f5(*args: *b = (1,)): pass
Traceback (most recent call last):
...
SyntaxError: invalid syntax

Other uses of starred expressions as annotations should fail

>>> def f6(x: *b): pass
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> def f7(x: *b = 1): pass
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> x: *b
Traceback (most recent call last):
...
SyntaxError: invalid syntax
>>> x: *b = 1
Traceback (most recent call last):
...
SyntaxError: invalid syntax
"""

__test__ = {'doctests' : doctests}

def test_main(verbose=False):
from test import support
from test import test_pep646_syntax
support.run_doctest(test_pep646_syntax, verbose)

if __name__ == "__main__":
test_main(verbose=True)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add support for PEP 646.
Loading