Skip to content

Commit 100072c

Browse files
pablogsalpablogsal
committed
fixup! pythongh-153569: Add tokenizer source and cursor primitives
1 parent 697e7dd commit 100072c

3 files changed

Lines changed: 28 additions & 30 deletions

File tree

‎Parser/tokenizer/cursor.c‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,10 @@ _PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno)
2525
_PyTok_Off start = cursor->line_end;
2626
_PyTok_Off end = source->len;
2727
if (lineno < source->nlines) {
28-
const char *newline = memchr(
29-
source->bytes + start, '\n', source->len - start);
30-
if (newline == NULL) {
31-
PyErr_SetString(PyExc_SystemError,
32-
"corrupt tokenizer source line index");
28+
end = _PyTok_SourceFindLineEnd(source, start);
29+
if (end < 0) {
3330
return -1;
3431
}
35-
end = newline - source->bytes + 1;
3632
}
3733
set_line(cursor, lineno, start, end);
3834
return 0;
@@ -74,14 +70,10 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
7470
_PyTok_Off start = offset - loc.byte_col;
7571
_PyTok_Off end = source->len;
7672
if (loc.lineno < source->nlines) {
77-
const char *newline = memchr(
78-
source->bytes + start, '\n', source->len - start);
79-
if (newline == NULL) {
80-
PyErr_SetString(PyExc_SystemError,
81-
"corrupt tokenizer source line index");
73+
end = _PyTok_SourceFindLineEnd(source, start);
74+
if (end < 0) {
8275
return -1;
8376
}
84-
end = newline - source->bytes + 1;
8577
}
8678
set_line(cursor, loc.lineno, start, end);
8779
cursor->pos = offset;

‎Parser/tokenizer/source.c‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -235,26 +235,18 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
235235
int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
236236
_PyTok_Off start = source->line_checkpoints[checkpoint];
237237
while (current < lineno) {
238-
const char *newline = memchr(
239-
source->bytes + start, '\n', source->len - start);
240-
if (newline == NULL) {
241-
PyErr_SetString(PyExc_SystemError,
242-
"corrupt tokenizer source line index");
238+
start = _PyTok_SourceFindLineEnd(source, start);
239+
if (start < 0) {
243240
return -1;
244241
}
245-
start = newline - source->bytes + 1;
246242
current++;
247243
}
248244
_PyTok_Off end = source->len;
249245
if (lineno < source->nlines) {
250-
const char *newline = memchr(
251-
source->bytes + start, '\n', source->len - start);
252-
if (newline == NULL) {
253-
PyErr_SetString(PyExc_SystemError,
254-
"corrupt tokenizer source line index");
246+
end = _PyTok_SourceFindLineEnd(source, start);
247+
if (end < 0) {
255248
return -1;
256249
}
257-
end = newline - source->bytes + 1;
258250
}
259251
*line = (_PyTok_Line){
260252
.start = start,
@@ -306,14 +298,10 @@ _PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset,
306298
int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
307299
_PyTok_Off start = source->line_checkpoints[checkpoint];
308300
while (lineno < source->nlines) {
309-
const char *newline = memchr(
310-
source->bytes + start, '\n', source->len - start);
311-
if (newline == NULL) {
312-
PyErr_SetString(PyExc_SystemError,
313-
"corrupt tokenizer source line index");
301+
_PyTok_Off end = _PyTok_SourceFindLineEnd(source, start);
302+
if (end < 0) {
314303
return -1;
315304
}
316-
_PyTok_Off end = newline - source->bytes + 1;
317305
if (offset < end ||
318306
(offset == end && affinity == _PYTOK_AFFINITY_LEFT)) {
319307
break;

‎Parser/tokenizer/source.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,22 @@ _PyTok_SpanIsValid(_PyTok_Span span)
6767
return span.start >= 0 && span.end >= span.start;
6868
}
6969

70+
static inline _PyTok_Off
71+
_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start)
72+
{
73+
if (source->bytes == NULL || start < 0 || start >= source->len) {
74+
PyErr_SetString(PyExc_SystemError,
75+
"corrupt tokenizer source line index");
76+
return -1;
77+
}
78+
const char *newline = memchr(
79+
source->bytes + start, '\n', source->len - start);
80+
if (newline == NULL) {
81+
PyErr_SetString(PyExc_SystemError,
82+
"corrupt tokenizer source line index");
83+
return -1;
84+
}
85+
return newline - source->bytes + 1;
86+
}
87+
7088
#endif

0 commit comments

Comments
 (0)