2828
2929TokenCollection = namedtuple (
3030 'TokenCollection' ,
31- 'pseudo_token single_quoted triple_quoted endpats fstring_pattern_map always_break_tokens' ,
31+ 'pseudo_token single_quoted triple_quoted endpats whitespace '
32+ 'fstring_pattern_map always_break_tokens' ,
3233)
3334
3435BOM_UTF8_STRING = BOM_UTF8 .decode ('utf-8' )
@@ -114,6 +115,7 @@ def _create_token_collection(version_info):
114115 # Note: we use unicode matching for names ("\w") but ascii matching for
115116 # number literals.
116117 Whitespace = r'[ \f\t]*'
118+ whitespace = _compile (Whitespace )
117119 Comment = r'#[^\r\n]*'
118120 Name = r'\w+'
119121
@@ -225,7 +227,7 @@ def _create_token_collection(version_info):
225227 pseudo_token_compiled = _compile (PseudoToken )
226228 return TokenCollection (
227229 pseudo_token_compiled , single_quoted , triple_quoted , endpats ,
228- fstring_pattern_map , ALWAYS_BREAK_TOKENS
230+ whitespace , fstring_pattern_map , ALWAYS_BREAK_TOKENS
229231 )
230232
231233
@@ -244,7 +246,7 @@ def _get_type_name(self, exact=True):
244246 return tok_name [self .type ]
245247
246248 def __repr__ (self ):
247- return ('TokenInfo(type=%s, string=%r, start =%r, prefix=%r)' %
249+ return ('TokenInfo(type=%s, string=%r, start_pos =%r, prefix=%r)' %
248250 self ._replace (type = self ._get_type_name ()))
249251
250252
@@ -354,7 +356,8 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
354356 token. This idea comes from lib2to3. The prefix contains all information
355357 that is irrelevant for the parser like newlines in parentheses or comments.
356358 """
357- pseudo_token , single_quoted , triple_quoted , endpats , fstring_pattern_map , always_break_tokens , = \
359+ pseudo_token , single_quoted , triple_quoted , endpats , whitespace , \
360+ fstring_pattern_map , always_break_tokens , = \
358361 _get_token_collection (version_info )
359362 paren_level = 0 # count parentheses
360363 indents = [0 ]
@@ -435,10 +438,14 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
435438
436439 pseudomatch = pseudo_token .match (line , pos )
437440 if not pseudomatch : # scan for tokens
438- txt = line [pos :]
439- if txt .endswith ('\n ' ):
441+ if line .endswith ('\n ' ):
440442 new_line = True
441- yield PythonToken (ERRORTOKEN , txt , (lnum , pos ), additional_prefix )
443+ match = whitespace .match (line , pos )
444+ pos = match .end ()
445+ yield PythonToken (
446+ ERRORTOKEN , line [pos :], (lnum , pos ),
447+ additional_prefix + match .group (0 )
448+ )
442449 additional_prefix = ''
443450 break
444451
0 commit comments