Support per-argument type comments ('long form')

This involved a complete refactor of the part of the grammar used to recognise formal parameters in function definitions. I renamed many of the rules involved. The rewrite always includes the comma following a parameter in that argument, so that we can also include the type comment, if any (which follows the comma). The final parameter may not be followed by a comma, so we allow the comma to be omitted, but only if a close parenthesis follows instead (preceded by the type comment, if any). Some additional complications are needed because the code generator doesn't actually support alternatives that may be empty. PS. I didn't refactor the corresponding rules for lambdas, since they don't have type comments.
gvanrossum · Apr 30, 2020 · Apr 26, 2020 · Apr 27, 2020 · Apr 28, 2020 · Apr 29, 2020
commit ed95a14898d5c56c0c6303ceede7d3c10ba5c7fc
diff --git a/Grammar/python.gram b/Grammar/python.gram
@@ -188,7 +188,7 @@ function_def[stmt_ty]:
     | function_def_raw
 
 function_def_raw[stmt_ty]:
-    | is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=annotation { z }] ':' tc=[func_type_comment] b=block {
+    | is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
         (is_async ? _Py_AsyncFunctionDef : _Py_FunctionDef)(n->v.Name.id,
                              (params) ? params : CHECK(_PyPegen_empty_arguments(p)),
                              b, NULL, a, NEW_TYPE_COMMENT(tc), EXTRA) }
@@ -200,35 +200,65 @@ func_type_comment[PyObject *]:
 params[arguments_ty]:
     | invalid_parameters
     | parameters
+
 parameters[arguments_ty]:
-    | a=slash_without_default b=[',' x=plain_names { x }] c=[',' y=names_with_default { y }] d=[',' z=[star_etc] { z }] {
+    | a=slash_no_default b=param_no_default* c=param_with_default* d=[star_etc] {
         _PyPegen_make_arguments(p, a, NULL, b, c, d) }
-    | a=slash_with_default b=[',' y=names_with_default { y }] c=[',' z=[star_etc] { z }] {
+    | a=slash_with_default b=param_with_default* c=[star_etc] {
         _PyPegen_make_arguments(p, NULL, a, NULL, b, c) }
-    | a=plain_names b=[',' y=names_with_default { y }] c=[',' z=[star_etc] { z }] {
+    | a=param_no_default+ b=param_with_default* c=[star_etc] {
         _PyPegen_make_arguments(p, NULL, NULL, a, b, c) }
-    | a=names_with_default b=[',' z=[star_etc] { z }] { _PyPegen_make_arguments(p, NULL, NULL, NULL, a, b)}
+    | a=param_with_default+ b=[star_etc] { _PyPegen_make_arguments(p, NULL, NULL, NULL, a, b)}
     | a=star_etc { _PyPegen_make_arguments(p, NULL, NULL, NULL, NULL, a) }
-slash_without_default[asdl_seq*]: a=plain_names ',' '/' { a }
-slash_with_default[SlashWithDefault*]: a=[n=plain_names ',' { n }] b=names_with_default ',' '/' {
-    _PyPegen_slash_with_default(p, a, b) }
+
+# Some duplication here because we can't write (',' | &')'),
+# which is because we don't support empty alternatives (yet).
+#
+slash_no_default[asdl_seq*]:
+    | a=param_no_default+ '/' ',' { a }
+    | a=param_no_default+ '/' &')' { a }
+slash_with_default[SlashWithDefault*]:
+    | a=param_no_default* b=param_with_default+ '/' ',' { _PyPegen_slash_with_default(p, a, b) }
+    | a=param_no_default* b=param_with_default+ '/' &')' { _PyPegen_slash_with_default(p, a, b) }
+
 star_etc[StarEtc*]:
-    | '*' a=plain_name b=name_with_optional_default* c=[',' d=kwds { d }] [','] {
+    | '*' a=param_no_default b=param_maybe_default* c=[kwds] {
         _PyPegen_star_etc(p, a, b, c) }
-    | '*' b=name_with_optional_default+ c=[',' d=kwds { d }] [','] {
+    | '*' ',' b=param_maybe_default+ c=[kwds] {
         _PyPegen_star_etc(p, NULL, b, c) }
-    | a=kwds [','] { _PyPegen_star_etc(p, NULL, NULL, a) }
-name_with_optional_default[NameDefaultPair*]:
-    | ',' a=plain_name b=['=' e=expression { e }] { _PyPegen_name_default_pair(p, a, b) }
-names_with_default[asdl_seq*]: a=','.name_with_default+ { a }
-name_with_default[NameDefaultPair*]:
-    | n=plain_name '=' e=expression { _PyPegen_name_default_pair(p, n, e) }
-plain_names[asdl_seq*] (memo): a=','.(plain_name !'=')+ { a }
-plain_name[arg_ty]:
-    | a=NAME b=[':' z=annotation { z }] { _Py_arg(a->v.Name.id, b, NULL, EXTRA) }
+    | a=kwds { _PyPegen_star_etc(p, NULL, NULL, a) }
+
 kwds[arg_ty]:
-    | '**' a=plain_name { a }
-annotation[expr_ty]: expression
+    | '**' a=param_no_default { a }
+
+# One parameter.  This *includes* a following comma and type comment.
+#
+# There are three styles:
+# - No default
+# - With default
+# - Maybe with default
+#
+# There are two alternative forms of each, to deal with type comments:
+# - Ends in a comma followed by an optional type comment
+# - No comma, optional type comment, must be followed by close paren
+# The latter form is for a final parameter without trailing comma.
+#
+param_no_default[arg_ty]:
+    | a=NAME b=annotation? ',' tc=TYPE_COMMENT? { _Py_arg(a->v.Name.id, b, tc, EXTRA) }
+    | a=NAME b=annotation? tc=TYPE_COMMENT? &')' { _Py_arg(a->v.Name.id, b, tc, EXTRA) }
+param_with_default[NameDefaultPair*]:
+    | a=NAME b=annotation? c=default ',' tc=TYPE_COMMENT? {
+        _PyPegen_name_default_pair(p, _Py_arg(a->v.Name.id, b, tc, EXTRA), c) }
+    | a=NAME b=annotation? c=default tc=TYPE_COMMENT? &')' {
+        _PyPegen_name_default_pair(p, _Py_arg(a->v.Name.id, b, tc, EXTRA), c) }
+param_maybe_default[NameDefaultPair*]:
+    | a=NAME b=annotation? c=default? ',' tc=TYPE_COMMENT? {
+        _PyPegen_name_default_pair(p, _Py_arg(a->v.Name.id, b, tc, EXTRA), c) }
+    | a=NAME b=annotation? c=default? tc=TYPE_COMMENT? &')' {
+        _PyPegen_name_default_pair(p, _Py_arg(a->v.Name.id, b, tc, EXTRA), c) }
+
+annotation[expr_ty]: ':' a=expression { a }
+default[expr_ty]: '=' a=expression { a }
 
 decorators[asdl_seq*]: a=('@' f=named_expression NEWLINE { f })+ { a }
 
@@ -567,7 +597,7 @@ invalid_comprehension:
     | ('[' | '(' | '{') '*' expression for_if_clauses {
         RAISE_SYNTAX_ERROR("iterable unpacking cannot be used in comprehension") }
 invalid_parameters:
-    | [plain_names ','] (slash_with_default | names_with_default) ',' plain_names {
+    | param_no_default* (slash_with_default | param_with_default+) param_no_default {
         RAISE_SYNTAX_ERROR("non-default argument follows default argument") }
 invalid_double_type_comments:
     | TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT {