Skip to content

Commit 3b11555

Browse files
pekkaklarckyanne
authored andcommitted
Change splitting statements in lexer.
Needed in two special cases: - Test/kw name in same row as first kw (or setting) - Inline IF Now top level algorigthm is simpler but actually splitting (i.e. adding EOS tokens) is pretty complicated.
1 parent b0848d1 commit 3b11555

3 files changed

Lines changed: 283 additions & 69 deletions

File tree

‎src/robot/parsing/lexer/lexer.py‎

Lines changed: 70 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -112,48 +112,87 @@ def get_tokens(self):
112112
return tokens
113113

114114
def _get_tokens(self, statements):
115-
name_and_eos_handler = self._get_name_and_eos_handler()
116-
for statement in statements:
117-
if not self._is_inline_if(statement):
118-
yield from name_and_eos_handler(statement)
119-
else:
120-
for part in self._split_inline_if(statement):
121-
yield from name_and_eos_handler(part)
122-
123-
def _get_name_and_eos_handler(self):
124115
# Setting local variables is performance optimization to avoid
125116
# unnecessary lookups and attribute access.
126117
if self.data_only:
127118
ignored_types = {None, Token.COMMENT_HEADER, Token.COMMENT}
128119
else:
129120
ignored_types = {None}
130-
name_types = (Token.TESTCASE_NAME, Token.KEYWORD_NAME)
131-
separator_type = Token.SEPARATOR
132-
eol_type = Token.EOL
133-
def name_and_eos_handler(statement):
134-
name_seen = False
135-
separator_after_name = None
136-
prev_token = None
121+
name_types = {Token.TESTCASE_NAME, Token.KEYWORD_NAME}
122+
if_type = Token.IF
123+
for statement in statements:
124+
eos_adder = None
125+
result = []
126+
append = result.append
137127
for token in statement:
138128
token_type = token.type
139129
if token_type in ignored_types:
140130
continue
141-
if name_seen:
142-
if token_type == separator_type:
143-
separator_after_name = token
144-
continue
145-
if token_type != eol_type:
146-
yield EOS.from_token(prev_token)
147-
if separator_after_name:
148-
yield separator_after_name
149-
name_seen = False
150131
if token_type in name_types:
151-
name_seen = True
152-
prev_token = token
153-
yield token
154-
if prev_token:
155-
yield EOS.from_token(prev_token)
156-
return name_and_eos_handler
132+
eos_adder = self._add_eos_to_name_statement
133+
if token_type == if_type:
134+
eos_adder = self._add_eos_to_if_statement
135+
append(token)
136+
if eos_adder:
137+
eos_adder(result)
138+
elif result:
139+
append(EOS.from_token(result[-1]))
140+
yield from result
141+
142+
def _add_eos_to_name_statement(self, statement):
143+
eol_type = Token.EOL
144+
separator_type = Token.SEPARATOR
145+
name_types = {Token.TESTCASE_NAME, Token.KEYWORD_NAME}
146+
name_seen = False
147+
eos_index = None
148+
for index, token in enumerate(statement):
149+
token_type = token.type
150+
if token.type in name_types:
151+
name_seen = True
152+
elif name_seen:
153+
if token_type == separator_type:
154+
eos_index = index
155+
elif token_type == eol_type:
156+
eos_index = None
157+
else:
158+
eos_index = eos_index or index
159+
break
160+
if eos_index:
161+
statement.insert(eos_index, EOS.from_token(statement[eos_index-1]))
162+
statement.append(EOS.from_token(statement[-1]))
163+
164+
def _add_eos_to_if_statement(self, statement):
165+
if_else_markers = {Token.IF: (False, True),
166+
Token.ELSE_IF: (True, True),
167+
Token.ELSE: (True, False)}
168+
normal_if_statement_types = {Token.IF, Token.ARGUMENT, # TODO: Continuation?
169+
Token.SEPARATOR, Token.EOL}
170+
inline_if = False
171+
added = 0
172+
add_after_arg = False
173+
for index, token in enumerate(statement[:]):
174+
token_type = token.type
175+
if token_type in if_else_markers:
176+
add_before, add_after_arg = if_else_markers[token_type]
177+
if add_before:
178+
statement.insert(index + added, EOS.from_token(token, before=True))
179+
added += 1
180+
if not add_after_arg:
181+
statement.insert(index + added + 1, EOS.from_token(token))
182+
added += 1
183+
elif token_type == Token.ARGUMENT and add_after_arg:
184+
statement.insert(index + added + 1, EOS.from_token(token))
185+
added += 1
186+
add_after_arg = False
187+
if token_type not in normal_if_statement_types:
188+
inline_if = True
189+
last = statement[-1]
190+
if not added:
191+
statement.append(EOS.from_token(last))
192+
if inline_if:
193+
statement.extend([EOS.from_token(last),
194+
END.from_token(last, virtual=True),
195+
EOS.from_token(last)])
157196

158197
def _split_trailing_commented_and_empty_lines(self, statement):
159198
lines = self._split_to_lines(statement)
@@ -192,20 +231,3 @@ def _tokenize_variables(self, tokens):
192231
for token in tokens:
193232
for t in token.tokenize_variables():
194233
yield t
195-
196-
def _split_inline_if(self, statement):
197-
yield from InlineIfLexer.split_statements(statement)
198-
yield [END.from_token(statement[-1])]
199-
200-
def _is_inline_if(self, statement):
201-
if not self._is_normal_if_header(statement):
202-
return False
203-
if_index = [s.type for s in statement].index(Token.IF)
204-
return len(statement[if_index:]) > 2
205-
206-
def _is_normal_if_header(self, statement):
207-
for token in statement:
208-
if token.type == Token.IF:
209-
return True
210-
return False
211-

‎src/robot/parsing/lexer/tokens.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -216,17 +216,23 @@ def __init__(self, lineno=-1, col_offset=-1):
216216
Token.__init__(self, Token.EOS, '', lineno, col_offset)
217217

218218
@classmethod
219-
def from_token(cls, token):
220-
return EOS(lineno=token.lineno, col_offset=token.end_col_offset)
219+
def from_token(cls, token, before=False):
220+
col_offset = token.col_offset if before else token.end_col_offset
221+
return EOS(token.lineno, col_offset)
221222

222223

223224
class END(Token):
224-
"""Token representing END token used to signify block ending."""
225+
"""Token representing END token used to signify block ending.
226+
227+
Virtual END tokens have '' as their value, with "real" END tokens the
228+
value is 'END'.
229+
"""
225230
__slots__ = []
226231

227-
def __init__(self, lineno=-1, col_offset=-1):
228-
Token.__init__(self, Token.END, '', lineno, col_offset)
232+
def __init__(self, lineno=-1, col_offset=-1, virtual=False):
233+
value = 'END' if not virtual else ''
234+
Token.__init__(self, Token.END, value, lineno, col_offset)
229235

230236
@classmethod
231-
def from_token(cls, token):
232-
return END(lineno=token.lineno, col_offset=token.end_col_offset)
237+
def from_token(cls, token, virtual=False):
238+
return END(token.lineno, token.end_col_offset, virtual)

0 commit comments

Comments
 (0)