@@ -112,48 +112,87 @@ def get_tokens(self):
112112 return tokens
113113
114114 def _get_tokens (self , statements ):
115- name_and_eos_handler = self ._get_name_and_eos_handler ()
116- for statement in statements :
117- if not self ._is_inline_if (statement ):
118- yield from name_and_eos_handler (statement )
119- else :
120- for part in self ._split_inline_if (statement ):
121- yield from name_and_eos_handler (part )
122-
123- def _get_name_and_eos_handler (self ):
124115 # Setting local variables is performance optimization to avoid
125116 # unnecessary lookups and attribute access.
126117 if self .data_only :
127118 ignored_types = {None , Token .COMMENT_HEADER , Token .COMMENT }
128119 else :
129120 ignored_types = {None }
130- name_types = (Token .TESTCASE_NAME , Token .KEYWORD_NAME )
131- separator_type = Token .SEPARATOR
132- eol_type = Token .EOL
133- def name_and_eos_handler (statement ):
134- name_seen = False
135- separator_after_name = None
136- prev_token = None
121+ name_types = {Token .TESTCASE_NAME , Token .KEYWORD_NAME }
122+ if_type = Token .IF
123+ for statement in statements :
124+ eos_adder = None
125+ result = []
126+ append = result .append
137127 for token in statement :
138128 token_type = token .type
139129 if token_type in ignored_types :
140130 continue
141- if name_seen :
142- if token_type == separator_type :
143- separator_after_name = token
144- continue
145- if token_type != eol_type :
146- yield EOS .from_token (prev_token )
147- if separator_after_name :
148- yield separator_after_name
149- name_seen = False
150131 if token_type in name_types :
151- name_seen = True
152- prev_token = token
153- yield token
154- if prev_token :
155- yield EOS .from_token (prev_token )
156- return name_and_eos_handler
132+ eos_adder = self ._add_eos_to_name_statement
133+ if token_type == if_type :
134+ eos_adder = self ._add_eos_to_if_statement
135+ append (token )
136+ if eos_adder :
137+ eos_adder (result )
138+ elif result :
139+ append (EOS .from_token (result [- 1 ]))
140+ yield from result
141+
142+ def _add_eos_to_name_statement (self , statement ):
143+ eol_type = Token .EOL
144+ separator_type = Token .SEPARATOR
145+ name_types = {Token .TESTCASE_NAME , Token .KEYWORD_NAME }
146+ name_seen = False
147+ eos_index = None
148+ for index , token in enumerate (statement ):
149+ token_type = token .type
150+ if token .type in name_types :
151+ name_seen = True
152+ elif name_seen :
153+ if token_type == separator_type :
154+ eos_index = index
155+ elif token_type == eol_type :
156+ eos_index = None
157+ else :
158+ eos_index = eos_index or index
159+ break
160+ if eos_index :
161+ statement .insert (eos_index , EOS .from_token (statement [eos_index - 1 ]))
162+ statement .append (EOS .from_token (statement [- 1 ]))
163+
164+ def _add_eos_to_if_statement (self , statement ):
165+ if_else_markers = {Token .IF : (False , True ),
166+ Token .ELSE_IF : (True , True ),
167+ Token .ELSE : (True , False )}
168+ normal_if_statement_types = {Token .IF , Token .ARGUMENT , # TODO: Continuation?
169+ Token .SEPARATOR , Token .EOL }
170+ inline_if = False
171+ added = 0
172+ add_after_arg = False
173+ for index , token in enumerate (statement [:]):
174+ token_type = token .type
175+ if token_type in if_else_markers :
176+ add_before , add_after_arg = if_else_markers [token_type ]
177+ if add_before :
178+ statement .insert (index + added , EOS .from_token (token , before = True ))
179+ added += 1
180+ if not add_after_arg :
181+ statement .insert (index + added + 1 , EOS .from_token (token ))
182+ added += 1
183+ elif token_type == Token .ARGUMENT and add_after_arg :
184+ statement .insert (index + added + 1 , EOS .from_token (token ))
185+ added += 1
186+ add_after_arg = False
187+ if token_type not in normal_if_statement_types :
188+ inline_if = True
189+ last = statement [- 1 ]
190+ if not added :
191+ statement .append (EOS .from_token (last ))
192+ if inline_if :
193+ statement .extend ([EOS .from_token (last ),
194+ END .from_token (last , virtual = True ),
195+ EOS .from_token (last )])
157196
158197 def _split_trailing_commented_and_empty_lines (self , statement ):
159198 lines = self ._split_to_lines (statement )
@@ -192,20 +231,3 @@ def _tokenize_variables(self, tokens):
192231 for token in tokens :
193232 for t in token .tokenize_variables ():
194233 yield t
195-
196- def _split_inline_if (self , statement ):
197- yield from InlineIfLexer .split_statements (statement )
198- yield [END .from_token (statement [- 1 ])]
199-
200- def _is_inline_if (self , statement ):
201- if not self ._is_normal_if_header (statement ):
202- return False
203- if_index = [s .type for s in statement ].index (Token .IF )
204- return len (statement [if_index :]) > 2
205-
206- def _is_normal_if_header (self , statement ):
207- for token in statement :
208- if token .type == Token .IF :
209- return True
210- return False
211-
0 commit comments