Skip to content

Commit b12e9d1

Browse files
authored
Update parso to 0.2.1 (microsoft#1841)
Fixes microsoft#1833 Fixes microsoft#1721
1 parent 33505d2 commit b12e9d1

8 files changed

Lines changed: 29 additions & 192 deletions

File tree

‎news/2 Fixes/1721.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix for intellisense failing when using the new `Outline` feature.

‎news/3 Code Health/1833.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Update `parso` package to 0.2.1.

‎pythonFiles/parso/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
from parso.utils import split_lines, python_bytes_to_unicode
4444

4545

46-
__version__ = '0.2.0'
46+
__version__ = '0.2.1'
4747

4848

4949
def parse(code=None, **kwargs):

‎pythonFiles/parso/grammar.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class Grammar(object):
2020
"""
2121
:py:func:`parso.load_grammar` returns instances of this class.
2222
23-
Creating custom grammars by calling this is not supported, yet.
23+
Creating custom none-python grammars by calling this is not supported, yet.
2424
"""
2525
#:param text: A BNF representation of your grammar.
2626
_error_normalizer_config = None
@@ -219,12 +219,13 @@ def load_grammar(**kwargs):
219219
version.
220220
221221
:param str version: A python version string, e.g. ``version='3.3'``.
222+
:param str path: A path to a grammar file
222223
"""
223-
def load_grammar(language='python', version=None):
224+
def load_grammar(language='python', version=None, path=None):
224225
if language == 'python':
225226
version_info = parse_version_string(version)
226227

227-
file = os.path.join(
228+
file = path or os.path.join(
228229
'python',
229230
'grammar%s%s.txt' % (version_info.major, version_info.minor)
230231
)

‎pythonFiles/parso/python/diff.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,9 @@ def _copy_nodes(self, tos, nodes, until_line, line_offset):
490490

491491
new_tos = tos
492492
for node in nodes:
493+
if node.start_pos[0] > until_line:
494+
break
495+
493496
if node.type == 'endmarker':
494497
# We basically removed the endmarker, but we are not allowed to
495498
# remove the newline at the end of the line, otherwise it's
@@ -501,8 +504,6 @@ def _copy_nodes(self, tos, nodes, until_line, line_offset):
501504
# Endmarkers just distort all the checks below. Remove them.
502505
break
503506

504-
if node.start_pos[0] > until_line:
505-
break
506507
# TODO this check might take a bit of time for large files. We
507508
# might want to change this to do more intelligent guessing or
508509
# binary search.

‎pythonFiles/parso/python/issue_list.txt‎

Lines changed: 0 additions & 176 deletions
This file was deleted.

‎pythonFiles/parso/python/tokenize.py‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@
2828

2929
TokenCollection = namedtuple(
3030
'TokenCollection',
31-
'pseudo_token single_quoted triple_quoted endpats fstring_pattern_map always_break_tokens',
31+
'pseudo_token single_quoted triple_quoted endpats whitespace '
32+
'fstring_pattern_map always_break_tokens',
3233
)
3334

3435
BOM_UTF8_STRING = BOM_UTF8.decode('utf-8')
@@ -114,6 +115,7 @@ def _create_token_collection(version_info):
114115
# Note: we use unicode matching for names ("\w") but ascii matching for
115116
# number literals.
116117
Whitespace = r'[ \f\t]*'
118+
whitespace = _compile(Whitespace)
117119
Comment = r'#[^\r\n]*'
118120
Name = r'\w+'
119121

@@ -225,7 +227,7 @@ def _create_token_collection(version_info):
225227
pseudo_token_compiled = _compile(PseudoToken)
226228
return TokenCollection(
227229
pseudo_token_compiled, single_quoted, triple_quoted, endpats,
228-
fstring_pattern_map, ALWAYS_BREAK_TOKENS
230+
whitespace, fstring_pattern_map, ALWAYS_BREAK_TOKENS
229231
)
230232

231233

@@ -244,7 +246,7 @@ def _get_type_name(self, exact=True):
244246
return tok_name[self.type]
245247

246248
def __repr__(self):
247-
return ('TokenInfo(type=%s, string=%r, start=%r, prefix=%r)' %
249+
return ('TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r)' %
248250
self._replace(type=self._get_type_name()))
249251

250252

@@ -354,7 +356,8 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
354356
token. This idea comes from lib2to3. The prefix contains all information
355357
that is irrelevant for the parser like newlines in parentheses or comments.
356358
"""
357-
pseudo_token, single_quoted, triple_quoted, endpats, fstring_pattern_map, always_break_tokens, = \
359+
pseudo_token, single_quoted, triple_quoted, endpats, whitespace, \
360+
fstring_pattern_map, always_break_tokens, = \
358361
_get_token_collection(version_info)
359362
paren_level = 0 # count parentheses
360363
indents = [0]
@@ -435,10 +438,14 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
435438

436439
pseudomatch = pseudo_token.match(line, pos)
437440
if not pseudomatch: # scan for tokens
438-
txt = line[pos:]
439-
if txt.endswith('\n'):
441+
if line.endswith('\n'):
440442
new_line = True
441-
yield PythonToken(ERRORTOKEN, txt, (lnum, pos), additional_prefix)
443+
match = whitespace.match(line, pos)
444+
pos = match.end()
445+
yield PythonToken(
446+
ERRORTOKEN, line[pos:], (lnum, pos),
447+
additional_prefix + match.group(0)
448+
)
442449
additional_prefix = ''
443450
break
444451

‎pythonFiles/parso/python/tree.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@
6060
_IMPORTS = set(['import_name', 'import_from'])
6161

6262

63-
6463
class DocstringMixin(object):
6564
__slots__ = ()
6665

@@ -133,7 +132,6 @@ def get_start_pos_of_prefix(self):
133132
return previous_leaf.end_pos
134133

135134

136-
137135
class _LeafWithoutNewlines(PythonLeaf):
138136
"""
139137
Simply here to optimize performance.
@@ -166,6 +164,10 @@ class EndMarker(_LeafWithoutNewlines):
166164
__slots__ = ()
167165
type = 'endmarker'
168166

167+
@utf8_repr
168+
def __repr__(self):
169+
return "<%s: prefix=%s>" % (type(self).__name__, repr(self.prefix))
170+
169171

170172
class Newline(PythonLeaf):
171173
"""Contains NEWLINE and ENDMARKER tokens."""
@@ -235,7 +237,6 @@ def get_definition(self, import_name_always=False):
235237
return None
236238

237239

238-
239240
class Literal(PythonLeaf):
240241
__slots__ = ()
241242

@@ -653,6 +654,7 @@ def annotation(self):
653654
except IndexError:
654655
return None
655656

657+
656658
class Lambda(Function):
657659
"""
658660
Lambdas are basically trimmed functions, so give it the same interface.

0 commit comments

Comments
 (0)