Name tokenizer bounds and derive byte lengths from their data

pablogsal · Oct 5, 2026 · Sep 24, 2026 · Sep 24, 2026 · Sep 24, 2026 · Sep 24, 2026
commit a29e2fd153d88d7a267f5b577cfaae6e86601885
diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c
@@ -38,6 +38,12 @@ static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
 {
+    const char multiple_lines[] = "a\nb\n";
+    const char first_line[] = "alpha\n";
+    const char second_line[] = "\xce\xb2\n";
+    const char expected[] = "alpha\n\xce\xb2\n";
+    const char tail[] = "tail";
+    const char terminated_line[] = "x\n";
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
@@ -49,13 +55,15 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
             _PyTok_SourceAppendLine(&source, "", 0) < 0,
             "accepted empty source line") < 0 ||
             check_system_error(
-                _PyTok_SourceAppendLine(&source, "a\nb\n", 4) < 0,
+                _PyTok_SourceAppendLine(
+                    &source, multiple_lines, sizeof(multiple_lines) - 1) < 0,
                 "accepted multiple source lines") < 0 ||
             check(_PyTok_SourceAppendLine(
-                      &source, "alpha\n", 6) == 0,
+                      &source, first_line, sizeof(first_line) - 1) == 0,
                   "wrong first source offset") < 0 ||
             check(_PyTok_SourceAppendLine(
-                      &source, "\xce\xb2\n", 3) == 6,
+                      &source, second_line, sizeof(second_line) - 1) ==
+                      (Py_ssize_t)sizeof(first_line) - 1,
                   "wrong second source offset") < 0) {
         goto error;
     }
@@ -68,16 +76,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
-    if (check(source.len == 9 &&
-                  memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0,
+    if (check(source.len == (Py_ssize_t)sizeof(expected) - 1 &&
+                  memcmp(source.bytes, expected, sizeof(expected)) == 0,
               "wrong source contents") < 0) {
         goto error;
     }
 
     _PyTok_SourceClear(&source);
-    if (_PyTok_SourceAppendLine(&source, "tail", 4) < 0 ||
+    if (_PyTok_SourceAppendLine(&source, tail, sizeof(tail) - 1) < 0 ||
             check_system_error(
-                _PyTok_SourceAppendLine(&source, "x\n", 2) < 0,
+                _PyTok_SourceAppendLine(
+                    &source, terminated_line, sizeof(terminated_line) - 1) < 0,
                 "appended after unterminated source line") < 0) {
         goto error;
     }
@@ -98,24 +107,34 @@ static PyObject *
 test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
                              PyObject *Py_UNUSED(args))
 {
+    enum { LINE_COUNT = 260 };
+    const char first_line[] = "x\n";
+    const char second_line[] = "y\n";
+    const char tail[] = "tail";
+    const char final_line[] = "z\n";
+    const _PyTok_Off first_batch_len = LINE_COUNT * (sizeof(first_line) - 1);
+    const _PyTok_Off second_batch_len = LINE_COUNT * (sizeof(second_line) - 1);
+    const _PyTok_Off discarded_len = first_batch_len + second_batch_len;
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
-    for (int i = 0; i < 260; i++) {
-        if (_PyTok_SourceAppendLine(&source, "x\n", 2) < 0) {
+    for (int i = 0; i < LINE_COUNT; i++) {
+        if (_PyTok_SourceAppendLine(&source, first_line, sizeof(first_line) - 1) < 0) {
             goto error;
         }
     }
     char *bytes = source.bytes;
     _PyTok_Off capacity = source.cap;
     _PyTok_SourceDiscard(&source);
-    if (check(source.base_offset == 520 && source.len == 0 &&
+    if (check(source.base_offset == first_batch_len && source.len == 0 &&
                   source.nlines == 0 && source.bytes == bytes &&
                   source.cap == capacity && source.bytes[0] == '\0',
               "discard did not preserve source allocation") < 0) {
         goto error;
     }
-    for (int i = 0; i < 260; i++) {
-        if (check(_PyTok_SourceAppendLine(&source, "y\n", 2) == 520 + 2 * i,
+    for (int i = 0; i < LINE_COUNT; i++) {
+        if (check(_PyTok_SourceAppendLine(
+                      &source, second_line, sizeof(second_line) - 1) ==
+                      first_batch_len + ((Py_ssize_t)sizeof(second_line) - 1) * i,
                   "wrong source offset after discard") < 0) {
             goto error;
         }
@@ -125,18 +144,22 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
         goto error;
     }
     _PyTok_SourceDiscard(&source);
-    if (check(_PyTok_SourceAppendLine(&source, "tail", 4) == 1040,
+    if (check(_PyTok_SourceAppendLine(
+                  &source, tail, sizeof(tail) - 1) == discarded_len,
               "wrong source offset after repeated discard") < 0) {
         goto error;
     }
     _PyTok_SourceDiscard(&source);
-    if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) == 1044,
+    if (check(_PyTok_SourceAppendLine(
+                  &source, final_line, sizeof(final_line) - 1) ==
+                  discarded_len + (Py_ssize_t)sizeof(tail) - 1,
               "cannot append after discarding unterminated line") < 0) {
         goto error;
     }
     _PyTok_SourceDiscard(&source);
     source.base_offset = PY_SSIZE_T_MAX - 1;
-    if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) < 0 &&
+    if (check(_PyTok_SourceAppendLine(
+                  &source, final_line, sizeof(final_line) - 1) < 0 &&
                   PyErr_ExceptionMatches(PyExc_MemoryError),
               "accepted overflowing logical source offset") < 0) {
         goto error;

@@ -8,6 +8,14 @@
 #include "helpers.h"
 #include "../lexer/state.h"
 
+#define UNICODE_BOM 0xFEFF
+#define UTF16_LE_BOM "\xff\xfe"
+#define UTF16_BE_BOM "\xfe\xff"
+#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1))
+#define UTF8_BOM "\xef\xbb\xbf"
+#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1))
+#define NORMALIZED_LINE_INITIAL_CAPACITY 256
+
 char *
 _PyTok_CopyBytes(const char *data, Py_ssize_t len)
 {
@@ -60,9 +68,9 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk,
         return -1;
     }
     if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 &&
-            PyUnicode_ReadChar(unicode, 0) == 0xFEFF) {
-        utf8 += 3;
-        utf8_len -= 3;
+            PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) {
+        utf8 += UTF8_BOM_LENGTH;
+        utf8_len -= UTF8_BOM_LENGTH;
     }
     chunk_release_data(chunk);
     chunk->owner = unicode;
@@ -72,8 +80,8 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk,
     return 0;
 }
 
-// The caller must provide len + 2 bytes: normalization can append a final
-// newline and always writes a NUL terminator.
+// The caller provides len + add_final_newline + 1 bytes for an optional
+// final newline and the NUL terminator.
 static void
 normalize_newlines_into(char *result, const char *data, Py_ssize_t len,
                         int preserve_crlf, int add_final_newline,
@@ -108,11 +116,11 @@ char *
 _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len,
                          Py_ssize_t *out_len)
 {
-    if (len > PY_SSIZE_T_MAX - 2) {
+    if (len < 0 || len == PY_SSIZE_T_MAX) {
         PyErr_NoMemory();
         return NULL;
     }
-    char *result = PyMem_Malloc((size_t)len + 2);
+    char *result = PyMem_Malloc((size_t)len + 1);
     if (result == NULL) {
         PyErr_NoMemory();
         return NULL;
@@ -152,11 +160,12 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next)
             return 0;
         }
     }
-    for (; i + 6 < len; i++) {
-        if (memcmp(line + i, "coding", 6) != 0) {
+    const Py_ssize_t marker_len = sizeof("coding") - 1;
+    for (; i < len - marker_len; i++) {
+        if (memcmp(line + i, "coding", marker_len) != 0) {
             continue;
         }
-        const char *cursor = line + i + 6;
+        const char *cursor = line + i + marker_len;
         if (*cursor != ':' && *cursor != '=') {
             continue;
         }
@@ -174,23 +183,26 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next)
             continue;
         }
         Py_ssize_t encoding_len = cursor - start;
-        char normalized[13];
-        int n;
-        for (n = 0; n < 12 && n < encoding_len; n++) {
+        // Include the longest recognized prefix and its NUL terminator.
+        char normalized[sizeof("iso-latin-1-")];
+        const Py_ssize_t prefix_len = Py_MIN(
+            encoding_len, (Py_ssize_t)sizeof(normalized) - 1);
+        Py_ssize_t n;
+        for (n = 0; n < prefix_len; n++) {
             normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]);
         }
         normalized[n] = '\0';
         const char *canonical = start;
         if (strcmp(normalized, "utf-8") == 0 ||
-                strncmp(normalized, "utf-8-", 6) == 0) {
+                strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) {
             canonical = "utf-8";
         }
         else if (strcmp(normalized, "latin-1") == 0 ||
                  strcmp(normalized, "iso-8859-1") == 0 ||
                  strcmp(normalized, "iso-latin-1") == 0 ||
-                 strncmp(normalized, "latin-1-", 8) == 0 ||
-                 strncmp(normalized, "iso-8859-1-", 11) == 0 ||
-                 strncmp(normalized, "iso-latin-1-", 12) == 0) {
+                 strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 ||
+                 strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 ||
+                 strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) {
             canonical = "iso-8859-1";
         }
         if (canonical != start) {
@@ -211,17 +223,15 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
                       const _PyTok_Chunk *second, int final,
                       Py_ssize_t *bom_len)
 {
-    int bom = first->len >= 3 &&
-        (unsigned char)first->data[0] == 0xEF &&
-        (unsigned char)first->data[1] == 0xBB &&
-        (unsigned char)first->data[2] == 0xBF;
-    *bom_len = bom ? 3 : 0;
+    int bom = first->len >= UTF8_BOM_LENGTH &&
+        memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0;
+    *bom_len = bom ? UTF8_BOM_LENGTH : 0;
 
     char *cookie = NULL;
     int scan_next = 0;
     int cookie_line = 1;
-    const char *first_data = first->data + (bom ? 3 : 0);
-    Py_ssize_t first_len = first->len - (bom ? 3 : 0);
+    const char *first_data = first->data + *bom_len;
+    Py_ssize_t first_len = first->len - *bom_len;
     if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) {
         return _PYTOK_ENCODING_ERROR;
     }
@@ -246,8 +256,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
     }
     if (bom && strcmp(cookie, "utf-8") != 0) {
         const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
-        const char *line_data = line->data + (cookie_line == 1 ? 3 : 0);
-        Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0);
+        const char *line_data = line->data + (cookie_line == 1 ? *bom_len : 0);
+        Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0);
         int end_col = (int)Py_MIN(line_len, INT_MAX);
         if (end_col > 0 && (line_data[end_col - 1] == '\n' ||
                             line_data[end_col - 1] == '\r')) {
@@ -325,14 +335,15 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
              (line[line_len - 1] == '\r' ||
               (line_len > 1 && line[line_len - 2] == '\r')));
         if (normalize) {
-            if (line_len > PY_SSIZE_T_MAX - 2) {
+            if (line_len > PY_SSIZE_T_MAX - add_newline - 1) {
                 PyErr_NoMemory();
                 tok->done = E_NOMEM;
                 goto error;
             }
             // Reserve space for an optional final '\n' and the NUL terminator.
-            Py_ssize_t needed = line_len + 2;
-            if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, 256) < 0) {
+            Py_ssize_t needed = line_len + add_newline + 1;
+            if (_PyTok_ReserveBuffer(&normalized, &capacity, needed,
+                                    NORMALIZED_LINE_INITIAL_CAPACITY) < 0) {
                 tok->done = E_NOMEM;
                 goto error;
             }
@@ -461,11 +472,9 @@ _PyTok_DecodeChunk(struct tok_state *tok, _PyTok_Chunk *chunk, int final)
         return 0;
     }
     int strip_bom = reader->kind == _PYTOK_READER_READLINE &&
-        chunk->len >= 2 &&
-        (((unsigned char)chunk->data[0] == 0xFF &&
-          (unsigned char)chunk->data[1] == 0xFE) ||
-         ((unsigned char)chunk->data[0] == 0xFE &&
-          (unsigned char)chunk->data[1] == 0xFF));
+        chunk->len >= UTF16_BOM_LENGTH &&
+        (memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 ||
+         memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0);
     PyObject *input;
     if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT &&
             PyBytes_Check(chunk->owner) &&

@@ -9,6 +9,8 @@
 #include "reader_internal.h"
 #include "../lexer/state.h"
 
+#define READER_BUFFER_GROWTH_FACTOR 2
+
 #ifdef HAVE_UNISTD_H
 #  include <unistd.h>
 #endif
@@ -67,11 +69,11 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed,
     // Grow geometrically to avoid reallocating for every longer line.
     Py_ssize_t cap = *capacity > 0 ? *capacity : initial_capacity;
     while (cap < needed) {
-        if (cap > PY_SSIZE_T_MAX / 2) {
+        if (cap > PY_SSIZE_T_MAX / READER_BUFFER_GROWTH_FACTOR) {
             cap = needed;
             break;
         }
-        cap *= 2;
+        cap *= READER_BUFFER_GROWTH_FACTOR;
     }
     char *resized = PyMem_Realloc(*buffer, cap);
     if (resized == NULL) {

@@ -3,6 +3,8 @@
 
 #include "Python.h"
 
+#define _PYTOK_ENCODING_COOKIE_MAX_LINES 2
+
 typedef enum {
     _PYTOK_READER_PREPARED,
     _PYTOK_READER_FILE,
@@ -46,11 +48,11 @@ typedef struct _PyTok_Reader {
 
     char *file_buffer;
     Py_ssize_t file_buffer_cap;
-    _PyTok_Chunk prefetched_lines[2];
+    _PyTok_Chunk prefetched_lines[_PYTOK_ENCODING_COOKIE_MAX_LINES];
 
     char *decoded;
     Py_ssize_t decoded_pos;
-    Py_ssize_t decoded_scan;
+    Py_ssize_t decoded_scan;  // First byte not yet scanned for a newline.
     Py_ssize_t decoded_len;
     Py_ssize_t decoded_cap;
     _PyTok_ReaderKind kind;

@@ -1,7 +1,10 @@
 #include "Python.h"
+#include "pycore_pymem.h"  // PYMEM_DEADBYTE
 
 #include "source.h"
 
+#define SOURCE_BUFFER_GROWTH_FACTOR 2
+
 void
 _PyTok_SourceInit(_PyTok_SourceText *source)
 {
@@ -35,11 +38,11 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
     }
     Py_ssize_t cap = source->cap > 0 ? source->cap : BUFSIZ;
     while (cap < needed) {
-        if (cap > PY_SSIZE_T_MAX / 2) {
+        if (cap > PY_SSIZE_T_MAX / SOURCE_BUFFER_GROWTH_FACTOR) {
             cap = needed;
             break;
         }
-        cap *= 2;
+        cap *= SOURCE_BUFFER_GROWTH_FACTOR;
     }
     char *bytes;
 #ifdef Py_DEBUG
@@ -57,7 +60,7 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
     }
 #ifdef Py_DEBUG
     if (source->bytes != NULL) {
-        memset(source->bytes, 0xDD, source->cap);
+        memset(source->bytes, PYMEM_DEADBYTE, source->cap);
         PyMem_Free(source->bytes);
     }
 #endif