Skip to content

Commit 5e6db31

Browse files
committed
Untokenize: An logically incorrect assert tested user input validity.
Replace it with correct logic that raises ValueError for bad input. Issues #8478 and #12691 reported the incorrect logic. Add an Untokenize test case and an initial test method.
1 parent cf62603 commit 5e6db31

2 files changed

Lines changed: 18 additions & 2 deletions

File tree

‎Lib/test/test_tokenize.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -638,7 +638,7 @@
638638
from test import support
639639
from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
640640
STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
641-
open as tokenize_open)
641+
open as tokenize_open, Untokenizer)
642642
from io import BytesIO
643643
from unittest import TestCase
644644
import os, sys, glob
@@ -1153,6 +1153,19 @@ def test_pathological_trailing_whitespace(self):
11531153
# See http://bugs.python.org/issue16152
11541154
self.assertExactTypeEqual('@ ', token.AT)
11551155

1156+
class UntokenizeTest(TestCase):
1157+
1158+
def test_bad_input_order(self):
1159+
u = Untokenizer()
1160+
u.prev_row = 2
1161+
u.prev_col = 2
1162+
with self.assertRaises(ValueError) as cm:
1163+
u.add_whitespace((1,3))
1164+
self.assertEqual(cm.exception.args[0],
1165+
'start (1,3) precedes previous end (2,2)')
1166+
self.assertRaises(ValueError, u.add_whitespace, (2,1))
1167+
1168+
11561169
__test__ = {"doctests" : doctests, 'decistmt': decistmt}
11571170

11581171
def test_main():
@@ -1162,6 +1175,7 @@ def test_main():
11621175
support.run_unittest(Test_Tokenize)
11631176
support.run_unittest(TestDetectEncoding)
11641177
support.run_unittest(TestTokenize)
1178+
support.run_unittest(UntokenizeTest)
11651179

11661180
if __name__ == "__main__":
11671181
test_main()

‎Lib/tokenize.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,9 @@ def __init__(self):
229229

230230
def add_whitespace(self, start):
231231
row, col = start
232-
assert row <= self.prev_row
232+
if row < self.prev_row or row == self.prev_row and col < self.prev_col:
233+
raise ValueError("start ({},{}) precedes previous end ({},{})"
234+
.format(row, col, self.prev_row, self.prev_col))
233235
col_offset = col - self.prev_col
234236
if col_offset:
235237
self.tokens.append(" " * col_offset)

0 commit comments

Comments
 (0)