Skip to content

gh-153569: hide tokenizer state behind token and source operations #156654

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pablogsal merged 4 commits into from
Sep 12, 2026
Merged

gh-153569: hide tokenizer state behind token and source operations #156654

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5caac0f
gh-153569: hide tokenizer state behind token and source operations
pablogsal Sep 6, 2026
97b6803
gh-153569: keep persistent scanner positions as source offsets
pablogsal Sep 6, 2026
1d56ef5
gh-153569: include token types and owned metadata in tokenizer results
pablogsal Sep 6, 2026
aef6c9e
gh-153569: keep character scanning inline with source offsets
pablogsal Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
gh-153569: keep persistent scanner positions as source offsets
  • Loading branch information
@pablogsal
pablogsal committed Sep 8, 2026
commit 97b6803a3c6156eb23c93e789ff9339010d1de6c
2 changes: 0 additions & 2 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,6 @@ PEGEN_OBJS= \
Parser/peg_api.o

TOKENIZER_OBJS= \
Parser/lexer/buffer.o \
Parser/lexer/lexer.o \
Parser/lexer/number.o \
Parser/lexer/state.o \
Expand All @@ -412,7 +411,6 @@ PEGEN_HEADERS= \
$(srcdir)/Parser/string_parser.h

TOKENIZER_HEADERS= \
Parser/lexer/buffer.h \
Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
Expand Down
1 change: 0 additions & 1 deletion PCbuild/_freeze_module.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,6 @@
<ClCompile Include="..\Parser\action_helpers.c" />
<ClCompile Include="..\Parser\string_parser.c" />
<ClCompile Include="..\Parser\token.c" />
<ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\lexer\state.c" />
<ClCompile Include="..\Parser\lexer\lexer.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
Expand Down
3 changes: 0 additions & 3 deletions PCbuild/_freeze_module.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,9 +469,6 @@
<ClCompile Include="..\Parser\lexer\string.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\buffer.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Source Files</Filter>
</ClCompile>
Expand Down
2 changes: 0 additions & 2 deletions PCbuild/pythoncore.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,6 @@
<ClInclude Include="..\Parser\lexer\state.h" />
<ClInclude Include="..\Parser\lexer\lexer.h" />
<ClInclude Include="..\Parser\lexer\lexer_internal.h" />
<ClInclude Include="..\Parser\lexer\buffer.h" />
<ClInclude Include="..\Parser\tokenizer\cursor.h" />
<ClInclude Include="..\Parser\tokenizer\reader.h" />
<ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
Expand Down Expand Up @@ -594,7 +593,6 @@
<ClCompile Include="..\Parser\lexer\lexer.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\tokenizer\cursor.c" />
<ClCompile Include="..\Parser\tokenizer\source.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
Expand Down
6 changes: 0 additions & 6 deletions PCbuild/pythoncore.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,9 +330,6 @@
<ClInclude Include="..\Parser\lexer\lexer_internal.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\lexer\buffer.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\tokenizer\cursor.h">
<Filter>Parser</Filter>
</ClInclude>
Expand Down Expand Up @@ -1364,9 +1361,6 @@
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\buffer.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\cursor.c">
<Filter>Parser</Filter>
</ClCompile>
Expand Down
29 changes: 0 additions & 29 deletions Parser/lexer/buffer.c
View file
Open in desktop

This file was deleted.

21 changes: 0 additions & 21 deletions Parser/lexer/buffer.h
View file
Open in desktop

This file was deleted.

48 changes: 24 additions & 24 deletions Parser/lexer/lexer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ _PyLexer_nextc(struct tok_state *tok)
tok->done = E_COLUMNOVERFLOW;
return EOF;
}
return Py_CHARMASK(*tok->cur++); /* Fast path */
return Py_CHARMASK(tok->source.bytes[tok->cur++ - tok->source.base_offset]); /* Fast path */
}
if (tok->done != E_OK) {
return EOF;
Expand All @@ -43,7 +43,7 @@ _PyLexer_nextc(struct tok_state *tok)
#if defined(Py_DEBUG)
if (tok->debug) {
fprintf(stderr, "line[%d] = ", tok->lineno);
_PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur);
_PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur), tok->inp - tok->cur);
fprintf(stderr, " tok->done = %d\n", tok->done);
}
#endif
Expand All @@ -53,7 +53,7 @@ _PyLexer_nextc(struct tok_state *tok)
}
tok->line_start = tok->cur;

if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start), tok->inp - tok->line_start)) {
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
tok->cur = tok->inp;
return EOF;
Expand All @@ -67,10 +67,10 @@ void
_PyLexer_backup(struct tok_state *tok, int c)
{
if (c != EOF) {
if (--tok->cur < tok->buf) {
if (--tok->cur < tok->buf_offset) {
Py_FatalError("tokenizer beginning of buffer");
}
if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
if ((int)(unsigned char)*_PyLexer_BufferPointer(tok, tok->cur) != Py_CHARMASK(c)) {
Py_FatalError("tok_backup: wrong character");
}
}
Expand All @@ -88,7 +88,7 @@ verify_identifier(struct tok_state *tok)
PyObject *s;
if (tok_failed(tok))
return 0;
s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
s = PyUnicode_DecodeUTF8(_PyLexer_BufferPointer(tok, tok->start), tok->cur - tok->start, NULL);
if (s == NULL) {
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
tok->done = E_DECODE;
Expand All @@ -113,7 +113,7 @@ verify_identifier(struct tok_state *tok)
tok->done = E_ERROR;
return 0;
}
tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
tok->cur = tok->start + PyBytes_GET_SIZE(s);
}
Py_DECREF(s);
if (Py_UNICODE_ISPRINTABLE(ch)) {
Expand Down Expand Up @@ -162,10 +162,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
int c;
int blankline, nonascii;

const char *p_start = NULL;
const char *p_end = NULL;
_PyTok_Off p_start = -1;
_PyTok_Off p_end = -1;
nextline:
tok->start = NULL;
tok->start = -1;
tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
blankline = 0;

Expand Down Expand Up @@ -271,7 +271,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token

tok->start = tok->cur;
tok->start_loc = (_PyTok_Loc){
tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) : -1};
tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1};

/* Return pending indents/dedents */
if (tok->pendin != 0) {
Expand All @@ -285,7 +285,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
}
else {
if (tok->tok_extra_tokens) {
p_start = tok->buf;
p_start = tok->buf_offset;
p_end = tok->cur;
}
tok->pendin--;
Expand All @@ -298,16 +298,16 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
tok_backup(tok, c);

again:
tok->start = NULL;
tok->start = -1;
/* Skip spaces */
do {
c = tok_nextc(tok);
} while (c == ' ' || c == '\t' || c == '\014');

/* Set start of current token */
tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
tok->start = tok->cur - 1;
tok->start_loc = (_PyTok_Loc){
tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) - 1 : -1};
tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) - 1 : -1};

/* Skip comment, unless it's a type comment */
if (c == '#') {
Expand All @@ -321,7 +321,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
}

if (current != NULL) {
const char *comment_end = tok->cur;
_PyTok_Off comment_end = tok->cur;
if (c == '\n' || c == '\r') {
comment_end--;
}
Expand All @@ -333,14 +333,14 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
}

if (tok->tok_extra_tokens) {
p = tok->start;
p = _PyLexer_BufferPointer(tok, tok->start);
}

if (tok->type_comments) {
p = tok->start;
p = _PyLexer_BufferPointer(tok, tok->start);
current_starting_col_offset = tok->start_loc.byte_col;
prefix = type_comment_prefix;
while (*prefix && p < tok->cur) {
while (*prefix && p < _PyLexer_BufferPointer(tok, tok->cur)) {
if (*prefix == ' ') {
while (*p == ' ' || *p == '\t') {
p++;
Expand Down Expand Up @@ -369,24 +369,24 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
/* A TYPE_IGNORE is "type: ignore" followed by the end of the token
* or anything ASCII and non-alphanumeric. */
is_type_ignore = (
tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
&& !(tok->cur > ignore_end
_PyLexer_BufferPointer(tok, tok->cur) >= ignore_end && memcmp(p, "ignore", 6) == 0
&& !(_PyLexer_BufferPointer(tok, tok->cur) > ignore_end
&& ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));

int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
int start_col_offset = is_type_ignore
? ignore_end_col_offset : current_starting_col_offset;
p_end = tok->cur;
if (is_type_ignore) {
p_start = ignore_end;
p_start = _PyLexer_BufferOffset(tok, ignore_end);

/* If this type ignore is the only thing on the line, consume the newline also. */
if (blankline) {
tok_nextc(tok);
tok->atbol = 1;
}
} else {
p_start = type_start;
p_start = _PyLexer_BufferOffset(tok, type_start);
}
_PyLexer_token_setup(tok, token, type, p_start, p_end);
token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
Expand All @@ -397,7 +397,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
}
if (tok->tok_extra_tokens) {
tok_backup(tok, c); /* don't eat the newline or EOF */
p_start = p;
p_start = _PyLexer_BufferOffset(tok, p);
p_end = tok->cur;
tok->comment_newline = blankline;
return MAKE_TOKEN(COMMENT);
Expand Down
2 changes: 1 addition & 1 deletion Parser/lexer/lexer_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ tok_failed(const struct tok_state *tok)
int _PyLexer_nextc(struct tok_state *);
void _PyLexer_backup(struct tok_state *, int);
int _PyLexer_record_ftstring_comment(
struct tok_state *, ftstring_state *, const char *, const char *);
struct tok_state *, ftstring_state *, _PyTok_Off, _PyTok_Off);
int _PyLexer_ftstring_punctuation(
struct tok_state *, ftstring_state *, struct token *, int);
int _PyLexer_close_ftstring_expr(
Expand Down
6 changes: 3 additions & 3 deletions Parser/lexer/number.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ int
_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
int leading_dot)
{
const char *p_start = NULL;
const char *p_end = NULL;
_PyTok_Off p_start = -1;
_PyTok_Off p_end = -1;

if (leading_dot) {
goto fraction;
Expand Down Expand Up @@ -214,7 +214,7 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
}
c = tok_nextc(tok);
}
char* zeros_end = tok->cur;
_PyTok_Off zeros_end = tok->cur;
if (Py_ISDIGIT(c)) {
nonzero = 1;
c = tok_decimal_tail(tok);
Expand Down
12 changes: 7 additions & 5 deletions Parser/lexer/state.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ _PyTokenizer_tok_new(void)
return NULL;
}

tok->buf = tok->cur = tok->inp = NULL;
tok->cur = tok->inp = 0;
tok->line_start = -1;
tok->fp_interactive = 0;
tok->interactive_src_start = NULL;
tok->interactive_src_end = NULL;
tok->start = NULL;
tok->start = -1;
tok->done = E_OK;
tok->fp = NULL;
tok->indent = 0;
Expand Down Expand Up @@ -139,12 +140,13 @@ _PyToken_Init(struct token *token) {
}

int
_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTok_Off start, _PyTok_Off end)
{
token->level = tok->level;
token->is_raw = 0;
token->span = _PyLexer_BufferSpan(tok, start, end);
if (start != NULL && end != NULL) {
assert((start == -1 && end == -1) || (start >= 0 && end >= start));
token->span = (_PyTok_Span){start, end};
if (start >= 0) {
token->start_loc = tok->start_loc;
token->end_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
}
Expand Down
Loading