Skip to content

Commit c4bce8d

Browse files
otethalotethal
committed
Fix test_eof
1 parent 4c75b11 commit c4bce8d

5 files changed

Lines changed: 51 additions & 5032 deletions

File tree

‎graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ def __str__(self) -> str:
244244
'_PyAST_Interactive ( a , p -> arena )': (1, 'factory.createInteractiveModule(a, $RANGE)'),
245245
'_PyAST_Lambda ( ( a ) ? a : CHECK ( arguments_ty , _PyPegen_empty_arguments ( p ) ) , b , EXTRA )': (1, 'factory.createLambda(a == null ? factory.emptyArguments() : a, b, $RANGE)'),
246246
'_PyAST_ListComp ( a , b , EXTRA )': (1, 'factory.createListComprehension(a, b, $RANGE)'),
247-
'CHECK_VERSION ( expr_ty , 8 , "Assignment expressions are" , _PyAST_NamedExpr ( CHECK ( expr_ty , _PyPegen_set_expr_context ( p , a , Store ) ) , b , EXTRA ) )': (1, 'checkVersion(8, "Assignment expressions are", factory.createNamedExp(this.check(this.setExprContext(a, ExprContextTy.Store)), b, $RANGE))'),
247+
'CHECK_VERSION ( expr_ty , 8 , "Assignment expressions are" , _PyAST_NamedExpr ( CHECK ( expr_ty , _PyPegen_set_expr_context ( p , a , Store ) ) , b , EXTRA ) )': (1, 'checkVersion(8, "Assignment expressions are", factory.createNamedExp(this.setExprContext(a, ExprContextTy.Store), b, $RANGE))'),
248248
'_PyAST_Nonlocal ( CHECK ( asdl_identifier_seq* , _PyPegen_map_names_to_ids ( p , a ) ) , EXTRA )': (1, 'factory.createNonLocal(extractNames(a), $RANGE)'),
249249
'_PyAST_Pass ( EXTRA )': (1, 'factory.createPass($RANGE)'),
250250
'_PyAST_Raise ( NULL , NULL , EXTRA )': (1, 'factory.createRaise(null, null, $RANGE)'),
@@ -1020,13 +1020,6 @@ def _set_up_token_end_metadata_extraction(self) -> None:
10201020
self.add_return("null")
10211021
self.print("}")
10221022

1023-
def _check_for_errors(self) -> None:
1024-
self.print("if (errorIndicator) {")
1025-
with self.indent():
1026-
self.remove_level();
1027-
self.print("return null;")
1028-
self.print("}")
1029-
10301023
def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None:
10311024
self.print("{")
10321025
with self.indent():
@@ -1064,7 +1057,6 @@ def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> N
10641057
memoize = self._should_memoize(node)
10651058

10661059
self.add_level()
1067-
self._check_for_errors()
10681060
self.print("int _mark = mark();")
10691061
self.print(f"Object _res = null;")
10701062
if memoize:
@@ -1095,7 +1087,6 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
10951087
is_repeat1 = node.name.startswith("_loop1")
10961088

10971089
self.add_level()
1098-
self._check_for_errors()
10991090
self.print("Object _res = null;")
11001091
self.print("int _mark = mark();")
11011092
if memoize:
@@ -1220,15 +1211,17 @@ def emit_default_action(self, is_gather: bool, node: Alt) -> None:
12201211
f"{self.local_variable_names[0]}, {self.local_variable_names[1]}, {element_type}.class);"
12211212
)
12221213
else:
1214+
node_str = str(node).replace('"', '\\"')
12231215
self.printDebug(
1224-
f'debugMessageln("Hit without action [%d:%d]: %s", _mark, mark(), "{node}");'
1216+
f'debugMessageln("Hit without action [%d:%d]: %s", _mark, mark(), "{node_str}");'
12251217
)
12261218
self.print(
12271219
f"_res = dummyName({', '.join(self.local_variable_names)});"
12281220
)
12291221
else:
1222+
node_str = str(node).replace('"', '\\"')
12301223
self.printDebug(
1231-
f'debugMessageln("Hit with default action [%d:%d]: %s", _mark, mark(), "{node}");'
1224+
f'debugMessageln("Hit with default action [%d:%d]: %s", _mark, mark(), "{node_str}");'
12321225
)
12331226
self.print(f"_res = {self.local_variable_names[0]};")
12341227

@@ -1293,7 +1286,6 @@ def visit_Alt(
12931286
else:
12941287
self.print(f"{{ // {node}")
12951288
with self.indent():
1296-
self._check_for_errors()
12971289
node_str = str(node).replace('"', '\\"')
12981290
self.printDebug(
12991291
f'debugMessageln("%d> {rulename}[%d-%d]: %s", level, _mark, mark(), "{node_str}");'

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java‎

Lines changed: 40 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
package com.oracle.graal.python.pegparser;
4242

4343
import static com.oracle.graal.python.pegparser.tokenizer.Token.Kind.DEDENT;
44+
import static com.oracle.graal.python.pegparser.tokenizer.Token.Kind.ENDMARKER;
4445
import static com.oracle.graal.python.pegparser.tokenizer.Token.Kind.ERRORTOKEN;
4546
import static com.oracle.graal.python.pegparser.tokenizer.Token.Kind.INDENT;
4647

@@ -127,11 +128,6 @@ public enum Flags {
127128

128129
private boolean parsingStarted;
129130

130-
/**
131-
* Indicates, whether there was found an error
132-
*/
133-
protected boolean errorIndicator = false;
134-
135131
private ExprTy.Name cachedDummyName;
136132

137133
protected final RuleResultCache<Object> cache = new RuleResultCache<>(this);
@@ -178,29 +174,27 @@ private static EnumSet<Flag> getTokenizerFlags(InputType type, EnumSet<Flags> pa
178174
public SSTNode parse() {
179175
SSTNode res = runParser(startRule);
180176
if (res == null) {
177+
Token lastToken = getFill() > 0 ? peekToken(getFill() - 1) : null;
181178
resetParserState();
182179
runParser(startRule);
183-
if (errorIndicator) {
184-
// shouldn't we return at least wrong AST based on a option?
185-
return null;
186-
}
187180
int fill = getFill();
188181
if (fill == 0) {
189182
throw raiseSyntaxError("error at start before reading any input");
190-
} else if (peekToken(fill - 1).type == Token.Kind.ERRORTOKEN && tokenizer.getDone() == Tokenizer.StatusCode.EOF) {
183+
}
184+
assert lastToken != null;
185+
if (lastToken.type == Token.Kind.ERRORTOKEN && tokenizer.getDone() == Tokenizer.StatusCode.EOF) {
191186
if (tokenizer.getParensNestingLevel() > 0) {
192187
throw raiseUnclosedParenthesesError();
193188
} else {
194189
throw raiseSyntaxError("unexpected EOF while parsing");
195190
}
191+
} else if (lastToken.type == INDENT) {
192+
throw raiseIndentationError("unexpected indent");
193+
} else if (lastToken.type == DEDENT) {
194+
throw raiseIndentationError("unexpected unindent");
196195
} else {
197-
if (peekToken(fill - 1).type == INDENT) {
198-
throw raiseIndentationError("unexpected indent");
199-
} else if (peekToken(fill - 1).type == DEDENT) {
200-
throw raiseIndentationError("unexpected unindent");
201-
} else {
202-
throw raiseSyntaxErrorKnownLocation(peekToken(fill - 1), "invalid syntax");
203-
}
196+
tokenizeFullSourceToCheckForErrors(lastToken);
197+
throw raiseSyntaxErrorKnownLocation(lastToken, "invalid syntax");
204198
}
205199
}
206200
if (startRule == InputType.SINGLE && tokenizer.isBadSingleStatement()) {
@@ -210,8 +204,36 @@ public SSTNode parse() {
210204
return res;
211205
}
212206

207+
/**
208+
* Equivalent of `_PyPegen_tokenize_full_source_to_check_for_errors`, but unlike cpython we
209+
* don't create the generic "invalid syntax" exception eagerly, so we pass the location of that
210+
* potential exception as an argument. This method either throws more specific exception from
211+
* the tokenizer, or returns normally in which case the caller is expected to throw the generic
212+
* exception.
213+
*/
214+
void tokenizeFullSourceToCheckForErrors(Token currentToken) {
215+
// We don't want to tokenize to the end for interactive input
216+
if (flags.contains(Flags.INTERACTIVE_TERMINAL)) {
217+
return;
218+
}
219+
while (true) {
220+
Token t = tokenizer.next();
221+
if (t.type == ENDMARKER) {
222+
break;
223+
}
224+
if (t.type == ERRORTOKEN) {
225+
if (tokenizer.getParensNestingLevel() != 0) {
226+
int errorLineNo = tokenizer.getParensLineNumberStack()[tokenizer.getParensNestingLevel() - 1];
227+
if (currentToken.getSourceRange().startLine > errorLineNo) {
228+
throw raiseUnclosedParenthesesError();
229+
}
230+
}
231+
throw tokenizerError(t);
232+
}
233+
}
234+
}
235+
213236
private void resetParserState() {
214-
errorIndicator = false;
215237
callInvalidRules = true;
216238
level = 0;
217239
cache.clear();
@@ -1342,7 +1364,6 @@ RuntimeException raiseSyntaxErrorKnownLocation(Token errorToken, String msg, Obj
13421364
* RAISE_ERROR_KNOWN_LOCATION
13431365
*/
13441366
RuntimeException raiseErrorKnownLocation(ParserCallbacks.ErrorType typeError, SourceRange where, String msg, Object... argument) {
1345-
errorIndicator = true;
13461367
throw callbacks.onError(typeError, where, msg, argument);
13471368
}
13481369

@@ -1517,16 +1538,6 @@ ModTy makeModule(StmtTy[] statements, SourceRange sourceRange) {
15171538
return factory.createModule(statements, comments.toArray(TypeIgnoreTy[]::new), sourceRange);
15181539
}
15191540

1520-
/**
1521-
* CHECK Simple check whether the node is not null.
1522-
*/
1523-
<T> T check(T node) {
1524-
if (node == null) {
1525-
errorIndicator = true;
1526-
}
1527-
return node;
1528-
}
1529-
15301541
<T> T checkVersion(int version, String msg, T node) {
15311542
checkVersion(version, msg);
15321543
return node;

0 commit comments

Comments
 (0)