Skip to content

Commit 4b0454f

Browse files
pablogsalpablogsal
committed
fixup! pythongh-153569: Add tokenizer source and cursor primitives
1 parent 100072c commit 4b0454f

4 files changed

Lines changed: 43 additions & 22 deletions

File tree

‎Modules/_testinternalcapi/tokenizer.c‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,12 +103,16 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
103103
check(line.start == 6 && line.end == 9 &&
104104
line.implicit_newline && !line.contains_nul,
105105
"wrong second source line") < 0 ||
106+
check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
107+
_PyTok_SourceLineIsImplicit(&source, 2),
108+
"wrong early implicit newline flags") < 0 ||
106109
check(_PyTok_SourceLine(&source, 3, &line) == 0,
107110
"cannot find third source line") < 0 ||
108111
check(line.contains_nul, "missing null byte flag") < 0 ||
109112
check(_PyTok_SourceLine(&source, final_line, &line) == 0,
110113
"cannot find final source line") < 0 ||
111-
check(line.implicit_newline,
114+
check(line.implicit_newline &&
115+
_PyTok_SourceLineIsImplicit(&source, final_line),
112116
"missing late implicit newline flag") < 0) {
113117
goto error;
114118
}
@@ -154,7 +158,11 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
154158
check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
155159
"cannot find virtual EOF line") < 0 ||
156160
check(line.start == source.len && line.end == source.len,
157-
"wrong virtual EOF line") < 0) {
161+
"wrong virtual EOF line") < 0 ||
162+
check(!_PyTok_SourceLineIsImplicit(&source, 0) &&
163+
!_PyTok_SourceLineIsImplicit(
164+
&source, final_line + 1),
165+
"virtual or invalid line is implicit") < 0) {
158166
goto error;
159167
}
160168

‎Parser/tokenizer/cursor.h‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,10 @@ typedef struct {
1212
int lineno;
1313
} _PyTok_Cursor;
1414

15+
/* Move to the start of a 1-based line. Both setters preserve the cursor on
16+
error. */
1517
PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int);
16-
/* A line-boundary offset selects the following line. */
18+
/* Move to an offset. A line boundary selects the following line. */
1719
PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
1820

1921
static inline void
@@ -24,7 +26,9 @@ _PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
2426
};
2527
}
2628

27-
/* EOF with pos before line_end reports byte-column overflow. */
29+
/* Read one byte from the current line, including its terminating newline.
30+
EOF marks the line boundary, not necessarily the end of the source. It is
31+
also returned if advancing would make the byte column unrepresentable. */
2832
static inline int
2933
_PyTok_CursorAdvance(_PyTok_Cursor *cursor)
3034
{
@@ -41,15 +45,18 @@ _PyTok_CursorAdvance(_PyTok_Cursor *cursor)
4145
return Py_CHARMASK(cursor->source->bytes[cursor->pos++]);
4246
}
4347

48+
/* Return the byte at a nonnegative distance within the current line, or EOF
49+
if the distance reaches or crosses the line boundary. */
4450
static inline int
4551
_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
4652
{
4753
assert(cursor->source != NULL);
4854
assert(cursor->pos >= cursor->line_start);
4955
assert(cursor->pos <= cursor->line_end);
5056
assert(cursor->line_end <= cursor->source->len);
51-
if ((size_t)distance >=
52-
(size_t)(cursor->line_end - cursor->pos)) {
57+
assert(distance >= 0);
58+
if (distance < 0 ||
59+
distance >= cursor->line_end - cursor->pos) {
5360
return EOF;
5461
}
5562
return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]);

‎Parser/tokenizer/source.c‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -75,14 +75,8 @@ reserve_checkpoints(_PyTok_SourceText *source, int needed)
7575
PyErr_NoMemory();
7676
return -1;
7777
}
78-
if ((size_t)cap > (size_t)PY_SSIZE_T_MAX /
79-
sizeof(*source->line_checkpoints)) {
80-
PyErr_NoMemory();
81-
return -1;
82-
}
83-
_PyTok_Off *checkpoints = PyMem_Realloc(
84-
source->line_checkpoints,
85-
(size_t)cap * sizeof(*source->line_checkpoints));
78+
_PyTok_Off *checkpoints = source->line_checkpoints;
79+
PyMem_Resize(checkpoints, _PyTok_Off, cap);
8680
if (checkpoints == NULL) {
8781
PyErr_NoMemory();
8882
return -1;
@@ -190,10 +184,11 @@ _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
190184
return source->bytes == NULL ? "" : source->bytes + span.start;
191185
}
192186

193-
static int
194-
line_is_implicit(const _PyTok_SourceText *source, int lineno)
187+
int
188+
_PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno)
195189
{
196-
if ((lineno - 1) / 8 >= source->implicit_cap) {
190+
if (lineno < 1 || lineno > source->nlines ||
191+
(lineno - 1) / 8 >= source->implicit_cap) {
197192
return 0;
198193
}
199194
return (source->implicit_lines[(lineno - 1) / 8] >>
@@ -251,7 +246,7 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
251246
*line = (_PyTok_Line){
252247
.start = start,
253248
.end = end,
254-
.implicit_newline = line_is_implicit(source, lineno),
249+
.implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno),
255250
.contains_nul = memchr(
256251
source->bytes + start, 0, end - start) != NULL,
257252
};

‎Parser/tokenizer/source.h‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@
55

66
typedef Py_ssize_t _PyTok_Off;
77

8+
/* Half-open byte offsets into a _PyTok_SourceText. */
89
typedef struct {
910
_PyTok_Off start;
1011
_PyTok_Off end;
1112
} _PyTok_Span;
1213

14+
/* Lines are 1-based and byte columns are 0-based. */
1315
typedef struct {
1416
int lineno;
1517
int byte_col;
@@ -20,6 +22,7 @@ typedef enum {
2022
_PYTOK_AFFINITY_RIGHT,
2123
} _PyTok_Affinity;
2224

25+
/* The half-open range includes the terminating newline when present. */
2326
typedef struct {
2427
_PyTok_Off start;
2528
_PyTok_Off end;
@@ -39,19 +42,27 @@ typedef struct {
3942
} _PyTok_SourceText;
4043

4144
PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
42-
/* Clear invalidates all cursors and span views for the source. */
45+
/* Clear invalidates all cursors, spans, and views for the source. */
4346
PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
44-
/* Append one logical line. The implicit_newline flag means that its newline
45-
terminator was synthesized. The input must not point into source storage. */
47+
/* Append one nonempty logical line and return its start offset. The input may
48+
contain one newline, as its final byte. An unterminated line must be the
49+
final line. implicit_newline means that the final newline was synthesized.
50+
The input must not point into source storage. */
4651
PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
4752
_PyTok_SourceText *source, const char *bytes, Py_ssize_t len,
4853
int implicit_newline);
4954
/* The returned view is invalidated by SourceAppendLine and SourceClear. */
5055
PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
5156
const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);
57+
/* Look up a 1-based line. Empty and newline-terminated sources have an empty
58+
virtual line at EOF. */
5259
PyAPI_FUNC(int) _PyTok_SourceLine(
5360
const _PyTok_SourceText *, int, _PyTok_Line *);
54-
/* At a line boundary, affinity selects the preceding or following line. */
61+
/* Return false for invalid line numbers and the virtual EOF line. */
62+
PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
63+
const _PyTok_SourceText *, int);
64+
/* At a line boundary, left affinity selects the preceding line at its end;
65+
right affinity selects the following line at byte column zero. */
5566
PyAPI_FUNC(int) _PyTok_SourceLocation(
5667
const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);
5768

0 commit comments

Comments
 (0)