pablogsal · Apr 23, 2020 · Apr 22, 2020 · Apr 23, 2020 · Apr 23, 2020 · Apr 23, 2020
diff --git a/Include/pegen_interface.h → Include/internal/pegen_interface.h b/Include/pegen_interface.h → Include/internal/pegen_interface.h
@@ -1,4 +1,3 @@
-#ifndef Py_LIMITED_API
 #ifndef Py_PEGENINTERFACE
 #define Py_PEGENINTERFACE
 #ifdef __cplusplus
@@ -29,4 +28,3 @@ PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFileObject(FILE *, PyObject *fi
 }
 #endif
 #endif /* !Py_PEGENINTERFACE*/
-#endif /* !Py_LIMITED_API */
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
@@ -545,10 +545,10 @@ def __hash__(self):
     def test_sys_flags(self):
         self.assertTrue(sys.flags)
         attrs = ("debug",
-                 "inspect", "interactive", "optimize", "use_peg",
-                 "dont_write_bytecode", "no_user_site", "no_site",
-                 "ignore_environment", "verbose", "bytes_warning", "quiet",
-                 "hash_randomization", "isolated", "dev_mode", "utf8_mode")
+                 "inspect", "interactive", "optimize", "dont_write_bytecode",
+                 "no_user_site", "no_site", "ignore_environment", "verbose",
+                 "bytes_warning", "quiet", "hash_randomization", "isolated",
+                 "dev_mode", "utf8_mode", "use_peg")
         for attr in attrs:
             self.assertTrue(hasattr(sys.flags, attr), attr)
             attr_type = bool if attr == "dev_mode" else int

diff --git a/Makefile.pre.in b/Makefile.pre.in
@@ -304,7 +304,7 @@ PEGEN_OBJS=		\
 
 
 PEGEN_HEADERS= \
-		$(srcdir)/Include/pegen_interface.h \
+		$(srcdir)/Include/internal/pegen_interface.h \
 		$(srcdir)/Parser/pegen/pegen.h \
 		$(srcdir)/Parser/pegen/parse_string.h
 

diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c
@@ -1,5 +1,5 @@
 #include <Python.h>
-#include <pegen_interface.h>
+#include "pegen_interface.h"
 
 PyObject *
 _Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds)

diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
@@ -161,6 +161,7 @@
     <ClInclude Include="..\Include\graminit.h" />
     <ClInclude Include="..\Include\grammar.h" />
     <ClInclude Include="..\Include\import.h" />
+    <ClInclude Include="..\Include\internal\pegen_interface.h" />
     <ClInclude Include="..\Include\internal\pycore_abstract.h" />
     <ClInclude Include="..\Include\internal\pycore_accu.h" />
     <ClInclude Include="..\Include\internal\pycore_atomic.h" />
@@ -213,7 +214,6 @@
     <ClInclude Include="..\Include\parsetok.h" />
     <ClInclude Include="..\Include\patchlevel.h" />
     <ClInclude Include="..\Include\picklebufobject.h" />
-    <ClInclude Include="..\Include\pegen_interface.h" />
     <ClInclude Include="..\Include\pyhash.h" />
     <ClInclude Include="..\Include\pyhash.h" />
     <ClInclude Include="..\Include\py_curses.h" />

diff --git a/Parser/pegen/peg_api.c b/Parser/pegen/peg_api.c
@@ -1,4 +1,4 @@
-#include <pegen_interface.h>
+#include "pegen_interface.h"
 
 #include "../tokenizer.h"
 #include "pegen.h"

diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
@@ -8,6 +8,9 @@
 static int
 init_normalization(Parser *p)
 {
+    if (p->normalize) {
+        return 0;
+    }
     PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
     if (!m)
     {
@@ -36,7 +39,7 @@ _PyPegen_new_identifier(Parser *p, char *n)
     if (!PyUnicode_IS_ASCII(id))
     {
         PyObject *id2;
-        if (!p->normalize && !init_normalization(p))
+        if (!init_normalization(p))
         {
             Py_DECREF(id);
             goto error;
@@ -88,6 +91,9 @@ static inline Py_ssize_t
 byte_offset_to_character_offset(PyObject *line, int col_offset)
 {
     const char *str = PyUnicode_AsUTF8(line);
+    if (!str) {
+        return 0;
+    }
     PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, NULL);
     if (!text) {
         return 0;
@@ -171,9 +177,10 @@ _PyPegen_get_expr_name(expr_ty e)
     }
 }
 
-static void
+static int
 raise_decode_error(Parser *p)
 {
+    assert(PyErr_Occurred());
     const char *errtype = NULL;
     if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {
         errtype = "unicode error";
@@ -197,6 +204,8 @@ raise_decode_error(Parser *p)
         Py_XDECREF(value);
         Py_XDECREF(tback);
     }
+
+    return -1;
 }
 
 static void
@@ -211,8 +220,7 @@ raise_tokenizer_init_error(PyObject *filename)
     PyErr_Fetch(&type, &value, &tback);
     errstr = PyObject_Str(value);
 
-    Py_INCREF(Py_None);
-    PyObject *tmp = Py_BuildValue("(OiiN)", filename, 0, -1, Py_None);
+    PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);
     if (!tmp) {
         goto error;
     }
@@ -337,9 +345,6 @@ tokenizer_error(Parser *p)
             errtype = PyExc_IndentationError;
             msg = "too many levels of indentation";
             break;
-        case E_DECODE:
-            raise_decode_error(p);
-            return -1;
         case E_LINECONT:
             msg = "unexpected character after line continuation character";
             break;
@@ -513,7 +518,12 @@ _PyPegen_fill_token(Parser *p)
     const char *start, *end;
     int type = PyTokenizer_Get(p->tok, &start, &end);
     if (type == ERRORTOKEN) {
-        return tokenizer_error(p);
+        if (p->tok->done == E_DECODE) {
+            return raise_decode_error(p);
+        }
+        else {
+            return tokenizer_error(p);
+        }
     }
     if (type == ENDMARKER && p->start_rule == Py_single_input && p->parsing_started) {
         type = NEWLINE; /* Add an extra newline */
@@ -530,13 +540,21 @@ _PyPegen_fill_token(Parser *p)
 
     if (p->fill == p->size) {
         int newsize = p->size * 2;
-        p->tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *));
-        if (p->tokens == NULL) {
-            PyErr_Format(PyExc_MemoryError, "Realloc tokens failed");
+        Token **new_tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *));
+        if (new_tokens == NULL) {
+            PyMem_Free(p->tokens);
+            p->tokens = NULL;
             return -1;
         }
+        else {
+            p->tokens = new_tokens;
+        }
         for (int i = p->size; i < newsize; i++) {
             p->tokens[i] = PyMem_Malloc(sizeof(Token));
+            if (p->tokens[i] == NULL) {
+                PyErr_NoMemory();
+                return -1;
+            }
             memset(p->tokens[i], '\0', sizeof(Token));
         }
         p->size = newsize;
@@ -566,8 +584,6 @@ _PyPegen_fill_token(Parser *p)
     t->end_lineno = p->starting_lineno + end_lineno;
     t->end_col_offset = p->tok->lineno == 1 ? p->starting_col_offset + end_col_offset : end_col_offset;
 
-    // if (p->fill % 100 == 0) fprintf(stderr, "Filled at %d: %s \"%s\"\n", p->fill,
-    // token_name(type), PyBytes_AsString(t->bytes));
     p->fill += 1;
     return 0;
 }
@@ -632,11 +648,9 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres)
             }
             p->mark = m->mark;
             *(void **)(pres) = m->node;
-            // fprintf(stderr, "%d < %d: memoized!\n", p->mark, p->fill);
             return 1;
         }
     }
-    // fprintf(stderr, "%d < %d: not memoized\n", p->mark, p->fill);
     return 0;
 }
 
@@ -678,13 +692,9 @@ _PyPegen_expect_token(Parser *p, int type)
     }
     Token *t = p->tokens[p->mark];
     if (t->type != type) {
-        // fprintf(stderr, "No %s at %d\n", token_name(type), p->mark);
         return NULL;
     }
     p->mark += 1;
-    // fprintf(stderr, "Got %s at %d: %s\n", token_name(type), p->mark,
-    // PyBytes_AsString(t->bytes));
-
     return t;
 }
 
@@ -866,10 +876,12 @@ void
 _PyPegen_Parser_Free(Parser *p)
 {
     Py_XDECREF(p->normalize);
-    for (int i = 0; i < p->size; i++) {
-        PyMem_Free(p->tokens[i]);
+    if (p->tokens) {
+        for (int i = 0; i < p->size; i++) {
+            PyMem_Free(p->tokens[i]);
+        }
+        PyMem_Free(p->tokens);
     }
-    PyMem_Free(p->tokens);
     PyMem_Free(p);
 }
 
@@ -878,8 +890,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int *errcode, PyArena
 {
     Parser *p = PyMem_Malloc(sizeof(Parser));
     if (p == NULL) {
-        PyErr_Format(PyExc_MemoryError, "Out of memory for Parser");
-        return NULL;
+        return (void *) PyErr_NoMemory();
     }
     assert(tok != NULL);
     p->tok = tok;
@@ -888,10 +899,14 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int *errcode, PyArena
     p->tokens = PyMem_Malloc(sizeof(Token *));
     if (!p->tokens) {
         PyMem_Free(p);
-        PyErr_Format(PyExc_MemoryError, "Out of memory for tokens");
-        return NULL;
+        return (void *) PyErr_NoMemory();
     }
     p->tokens[0] = PyMem_Malloc(sizeof(Token));
+    if (!p->tokens) {
+        PyMem_Free(p->tokens);
+        PyMem_Free(p);
+        return (void *) PyErr_NoMemory();
+    }
     memset(p->tokens[0], '\0', sizeof(Token));
     p->mark = 0;
     p->fill = 0;
@@ -1177,7 +1192,7 @@ _PyPegen_seq_count_dots(asdl_seq *seq)
                 number_of_dots += 1;
                 break;
             default:
-                assert(current_expr->type == ELLIPSIS || current_expr->type == DOT);
+                Py_UNREACHABLE();
         }
     }
 

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
@@ -29,7 +29,7 @@
 #include "ast.h"                  // PyAST_FromNodeObject()
 #include "marshal.h"              // PyMarshal_ReadLongFromFile()
 
-#include <pegen_interface.h>      // PyPegen_ASTFrom*
+#include "pegen_interface.h"      // PyPegen_ASTFrom*
 
 #ifdef MS_WINDOWS
 #  include "malloc.h"             // alloca()

diff --git a/Python/sysmodule.c b/Python/sysmodule.c
@@ -2427,7 +2427,6 @@ static PyStructSequence_Field flags_fields[] = {
     {"inspect",                 "-i"},
     {"interactive",             "-i"},
     {"optimize",                "-O or -OO"},
-    {"use_peg",                 "-p old or -p new"},
     {"dont_write_bytecode",     "-B"},
     {"no_user_site",            "-s"},
     {"no_site",                 "-S"},
@@ -2441,6 +2440,7 @@ static PyStructSequence_Field flags_fields[] = {
     {"isolated",                "-I"},
     {"dev_mode",                "-X dev"},
     {"utf8_mode",               "-X utf8"},
+    {"use_peg",                 "-X oldparser"},
     {0}
 };