Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 144 additions & 21 deletions src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,65 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
// then-branch closes the ternary and must not be taken as the JSON path operator
private int ternaryThenBranchDepth = 0;

// Keep this routing check aligned with SingleStatement, including aliases and dialect gates.
// It does not validate syntax that the grammar captures as opaque.
private boolean isStatementStartAhead() {
Token next = getToken(1);
switch (next.kind) {
case K_SELECT: case K_WITH: case K_VALUES: case K_VALUE: case K_FROM:
case K_PIVOT: case K_UNPIVOT: case OPENING_BRACKET:
case K_INSERT: case K_UPDATE: case K_DELETE: case K_MERGE:
case K_UPSERT: case K_REPLACE: case K_ALTER: case K_RENAME:
case K_CREATE: case K_DROP: case K_ANALYZE: case K_TRUNCATE:
case K_EXEC: case K_EXECUTE: case K_CALL: case K_SET: case K_RESET:
case K_SHOW: case K_REFRESH: case K_USE: case K_BEGIN: case K_START:
case K_SAVEPOINT: case K_ROLLBACK: case K_COMMIT: case K_COMMENT:
case K_DESCRIBE: case K_DESC: case K_EXPLAIN: case K_SUMMARIZE: case K_DECLARE: case K_GRANT:
case K_PRAGMA: case K_INSTALL: case K_LOAD: case K_FORCE:
case K_ATTACH: case K_DETACH: case K_CONNECT: case K_DISCONNECT:
case K_PREPARE: case K_DEALLOCATE: case K_COPY: case K_ASSERT:
case K_PURGE: case K_SESSION: case K_BRANCH: case K_LOCK: case K_IF: case K_TABLE:
return true;
case K_DO:
return Dialect.POSTGRESQL.name().equals(getAsString(Feature.dialect));
case K_IMPORT: case K_EXPORT:
return Dialect.EXASOL.name().equals(getAsString(Feature.dialect))
|| next.kind == K_EXPORT && getToken(2).kind == K_DATA;
default:
return "NOTIFY".equalsIgnoreCase(next.image)
|| "REVOKE".equalsIgnoreCase(next.image)
|| "RELEASE".equalsIgnoreCase(next.image);
}
}

private boolean isInformixIsolationLevel(Token token) {
return "DIRTY".equalsIgnoreCase(token.image) || "COMMITTED".equalsIgnoreCase(token.image)
|| "CURSOR".equalsIgnoreCase(token.image) || token.kind == K_REPEATABLE;
}

private boolean isOpaqueStatementAhead() {
int kind = getToken(1).kind;
return kind != EOF && kind != ST_SEMICOLON && kind != K_END && !isStatementStartAhead();
}

private void captureOpaqueStatements(Statements statements) throws ParseException {
while (getAsBoolean(Feature.allowUnsupportedStatements) && isOpaqueStatementAhead()) {
statements.add(UnsupportedStatement());
while (getToken(1).kind == ST_SEMICOLON) {
getNextToken();
}
}
}

private void requireStatementBoundary() throws ParseException {
int kind = getToken(1).kind;
if (kind != EOF && kind != ST_SEMICOLON
&& (kind == K_FROM || kind == K_PIVOT || kind == K_UNPIVOT || !isStatementStartAhead())) {
throw new ParseException("Unexpected token after a supported statement: " + getToken(1).image);
}
}


// depth of DuckDB MAP keys: the following ":" separates a key from its value
// and must not be taken as the JSON path operator
private int mapKeyDepth = 0;
Expand Down Expand Up @@ -3303,11 +3362,11 @@ Statement Statement() #Statement:
( <ST_SEMICOLON> )* <EOF>
)
|
LOOKAHEAD( { stm==null && getAsBoolean(Feature.allowUnsupportedStatements) } ) stm = UnsupportedStatement()
LOOKAHEAD( { stm==null && getAsBoolean(Feature.allowUnsupportedStatements) && isOpaqueStatementAhead() } )
stm = UnsupportedStatement() ( <ST_SEMICOLON> )* <EOF>
} catch (ParseException ex) {
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
stm = new UnsupportedStatement( stm.toString(), error_skipto(ST_SEMICOLON) );
} else if ( errorRecovery ) {
if ( errorRecovery ) {
ifElseStatement = null;
parseErrors.add(ex);
error_skipto(ST_SEMICOLON);
stm = null;
Expand Down Expand Up @@ -3392,6 +3451,16 @@ Statement SingleStatement() :
&& !"=".equals(getToken(3).image) && !".".equals(getToken(3).image) })
stm = SetIdentityInsert()
|
LOOKAHEAD({ getAsBoolean(Feature.allowUnsupportedStatements)
&& getToken(1).kind == K_SET && getToken(2).kind == S_IDENTIFIER
&& !"=".equals(getToken(3).image) && !".".equals(getToken(3).image)
&& !":=".equals(getToken(3).image)
&& ("IDENTITY_INSERT".equalsIgnoreCase(getToken(2).image)
|| "ISOLATION".equalsIgnoreCase(getToken(2).image)
&& (isInformixIsolationLevel(getToken(3))
|| getToken(3).kind == K_TO && isInformixIsolationLevel(getToken(4)))) })
stm = OpaqueSetStatement()
|
stm = Set()
|
stm = Reset()
Expand Down Expand Up @@ -3614,11 +3683,28 @@ Statements Statements() #Statements: {
Expression condition;
}
{
{
// Opaque roots can be first; capture them before known grammar lookahead.
while (getToken(1).kind == ST_SEMICOLON) {
getNextToken();
}
captureOpaqueStatements(stmts);
if (getToken(1).kind == EOF) {
getNextToken();
return stmts;
}
}
(
(
( <ST_SEMICOLON> )*
{
captureOpaqueStatements(stmts);
if (getToken(1).kind == EOF) {
getNextToken();
return stmts;
}
}

// todo: allow also first statement to be an `UnsupportedStatement`
try {
(
<K_IF> condition=Condition()
Expand All @@ -3628,7 +3714,7 @@ Statements Statements() #Statements: {
<K_ELSE> ( stm2 = SingleStatement() | stm2 = Block() ) { ifElseStatement.setElseStatement(stm2); }
]

{ stmts.add( ifElseStatement ); }
{ requireStatementBoundary(); stmts.add( ifElseStatement ); }

[
LOOKAHEAD(2) <ST_SEMICOLON>
Expand All @@ -3651,12 +3737,7 @@ Statements Statements() #Statements: {
}

} catch (ParseException ex) {
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
UnsupportedStatement unsupportedStatement = new UnsupportedStatement( stm!=null ? stm.toString() : "", error_skipto(ST_SEMICOLON) );
if (!unsupportedStatement.isEmpty()) {
stmts.add( unsupportedStatement );
}
} else if ( errorRecovery ) {
if ( errorRecovery ) {
parseErrors.add(ex);
error_skipto(ST_SEMICOLON);
stmts.add( null );
Expand All @@ -3669,6 +3750,13 @@ Statements Statements() #Statements: {

( LOOKAHEAD(2)
( <ST_SEMICOLON> )*
{
captureOpaqueStatements(stmts);
if (getToken(1).kind == EOF) {
getNextToken();
return stmts;
}
}
try {
(
<K_IF> condition=Condition()
Expand All @@ -3678,20 +3766,15 @@ Statements Statements() #Statements: {
<K_ELSE> ( stm2 = SingleStatement() | stm2 = Block() ) { ifElseStatement.setElseStatement(stm2); }
]

{ stmts.add( ifElseStatement ); }
{ requireStatementBoundary(); stmts.add( ifElseStatement ); }
)
|
(
stm = SingleStatement()
| stm = Block()
) { stmts.add(stm); stm=null; }
) { requireStatementBoundary(); stmts.add(stm); stm=null; }
} catch (ParseException ex) {
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
UnsupportedStatement unsupportedStatement = new UnsupportedStatement( stm!=null ? stm.toString() : "" , error_skipto(ST_SEMICOLON) );
if (!unsupportedStatement.isEmpty()) {
stmts.add( unsupportedStatement );
}
} else if ( errorRecovery ) {
if ( errorRecovery ) {
parseErrors.add(ex);
error_skipto(ST_SEMICOLON);
stmts.add( null );
Expand All @@ -3709,7 +3792,7 @@ Statements Statements() #Statements: {
)*

[
LOOKAHEAD( { getAsBoolean(Feature.allowUnsupportedStatements) } )
LOOKAHEAD( { getAsBoolean(Feature.allowUnsupportedStatements) && isOpaqueStatementAhead() } )
stm = UnsupportedStatement()
{
if (!( (UnsupportedStatement) stm).isEmpty()) {
Expand All @@ -3726,6 +3809,19 @@ Statements Statements() #Statements: {
}
}

JAVACODE
UnsupportedStatement captureStatementTokens(Token statementStart) {
List<String> tokenImages = new ArrayList<String>();
Token statementEnd = getToken(0);
for (Token t = statementStart.next; ; t = t.next) {
tokenImages.add(t.image);
if (t == statementEnd) {
break;
}
}
return new UnsupportedStatement(tokenImages);
}

JAVACODE
List<String> error_skipto(int kind) {
ArrayList<String> tokenImages = new ArrayList<String>();
Expand Down Expand Up @@ -4754,6 +4850,33 @@ SessionStatement SessionStatement():
}
}

/** Preserves existing opaque SET forms while requiring their complete syntax. */
private UnsupportedStatement OpaqueSetStatement(): {
Token statementStart = getToken(0);
SetIdentityInsertStatement identityInsert;
boolean repeatableRead = false;
}
{
(
LOOKAHEAD({ "IDENTITY_INSERT".equalsIgnoreCase(getToken(2).image) })
identityInsert = SetIdentityInsert()
|
<K_SET> AccessKeyword("ISOLATION") [ <K_TO> ]
(
LOOKAHEAD({ isKeywordAhead("DIRTY") }) AccessKeyword("DIRTY") <K_READ>
[ LOOKAHEAD({ getToken(1).kind == K_WITH && "WARNING".equalsIgnoreCase(getToken(2).image) })
<K_WITH> AccessKeyword("WARNING") ]
| LOOKAHEAD({ isKeywordAhead("COMMITTED") }) AccessKeyword("COMMITTED") <K_READ>
[ <K_LAST> AccessKeyword("COMMITTED") ]
| LOOKAHEAD({ isKeywordAhead("CURSOR") }) AccessKeyword("CURSOR") AccessKeyword("STABILITY")
| <K_REPEATABLE> <K_READ> { repeatableRead = true; }
)
[ LOOKAHEAD({ isKeywordAhead("RETAIN") }) AccessKeyword("RETAIN") <K_UPDATE> AccessKeyword("LOCKS")
{ requireAccessSyntax(!repeatableRead, "REPEATABLE READ cannot retain update locks"); } ]
)
{ return captureStatementTokens(statementStart); }
}

SetStatement Set(): {
SetStatement set;
}
Expand Down
19 changes: 16 additions & 3 deletions src/site/sphinx/usage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -782,7 +782,12 @@ open. These empty-input results replace the previous null returns of these metho
By default a syntax error aborts the whole parse. Two features let a script survive one bad statement:

- ``parser.withErrorRecovery(true)`` skips to the next statement separator and returns an empty statement.
- ``parser.withUnsupportedStatements(true)`` returns an ``UnsupportedStatement`` holding the raw text instead — though the **first** statement must be a regular one.
- ``parser.withUnsupportedStatements(true)`` captures an unrecognised statement root as
``UnsupportedStatement``; it may be the first statement. Failures in recognised statement
grammar still throw, or use ``withErrorRecovery(true)`` and record the error. Thus
``SELECT * FROM``, ``INSERT INTO t (a`` and ``UPDATE t SET`` do not become opaque statements.
The two options serve different purposes; enabling unsupported capture does not suppress
recognised syntax errors.

.. code-block:: java
:caption: Error Recovery
Expand All @@ -802,19 +807,27 @@ By default a syntax error aborts the whole parse. Two features let a script surv
:caption: Unsupported Statement

Statements statements = CCJSqlParserUtil.parseStatements(
"select * from mytable; select from; select * from mytable2; select 4;"
"select * from mytable; shutdown defrag; select * from mytable2; select 4;"
, parser -> parser.withUnsupportedStatements() );

// 4 statements with one Unsupported Statement holding the content
assertEquals(4, statements.size());
assertInstanceOf(UnsupportedStatement.class, statements.get(1));
assertEquals("select from", statements.get(1).toString());
assertEquals("shutdown defrag", statements.get(1).toString());

// no errors records, because a statement has been returned
assertEquals(0, parser.getParseErrors().size());

.. note::

Opaque capture is not SQL validation. An unrecognised root or an existing opaque
grammar branch cannot establish validity in an unknown dialect. Callers that need
validity must validate against that dialect. Recognised syntax failures formerly
captured by the generic fallback now throw or produce an error-recovery placeholder;
partial IF results and typed prefixes with invalid trailing clauses are not published
as successful statements. ``SingleStatement()`` remains an incremental entry point;
use ``Statement()`` or ``Statements()`` to check complete statement boundaries.

An ``UnsupportedStatement`` is reported as ``OPAQUE`` by :ref:`Classify a Statement` — nothing about its effects is knowable.


Expand Down
21 changes: 15 additions & 6 deletions src/test/java/net/sf/jsqlparser/statement/StatementsTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,17 @@
*/
package net.sf.jsqlparser.statement;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrowsExactly;

import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParser;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.parser.ParseException;
import net.sf.jsqlparser.parser.StringProvider;
import net.sf.jsqlparser.statement.select.Select;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;

import net.sf.jsqlparser.test.TestUtils;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -93,7 +94,7 @@ public void testStatementsErrorRecovery3() throws JSQLParserException, ParseExce
@Test
public void testStatementsErrorRecovery4() throws JSQLParserException {
Statements statements = CCJSqlParserUtil.parseStatements(
"select * from mytable; select from; select * from mytable2; select 4 from dual;",
"select * from mytable; shutdown defrag; select * from mytable2; select 4 from dual;",
parser -> parser.withUnsupportedStatements());

assertEquals(4, statements.size());
Expand All @@ -103,6 +104,14 @@ public void testStatementsErrorRecovery4() throws JSQLParserException {
assertInstanceOf(Select.class, statements.get(2));
assertInstanceOf(Select.class, statements.get(3));

TestUtils.assertStatementCanBeDeparsedAs(statements.get(1), "select from", true);
TestUtils.assertStatementCanBeDeparsedAs(statements.get(1), "shutdown defrag", true);
}

@Test
void malformedKnownStatementDoesNotUseUnsupportedCapture() {
assertThrowsExactly(JSQLParserException.class, () -> CCJSqlParserUtil.parseStatements(
"select * from mytable; select from; select * from mytable2; select 4 from dual;",
parser -> parser.withUnsupportedStatements()));
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*-
* #%L
* JSQLParser library
* %%
* Copyright (C) 2004 - 2026 JSQLParser
* %%
* Dual licensed under GNU LGPL 2.1 or Apache License 2.0
* #L%
*/
package net.sf.jsqlparser.statement;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrowsExactly;

import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParser;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.select.Select;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

class UnsupportedIfBoundaryTest {
@ParameterizedTest
@ValueSource(strings = {"IF x > 0 SELECT 1 WHERE", "IF x > 0 SELECT * FROM"})
void malformedIfBodyCannotBecomeAnOpaqueSuffix(String sql) {
for (boolean unsupported : new boolean[] {false, true}) {
for (String prefix : new String[] {"", "SELECT 0;"}) {
assertThrowsExactly(JSQLParserException.class,
() -> CCJSqlParserUtil.parseStatements(prefix + sql + ";SELECT 2;",
p -> p.withUnsupportedStatements(unsupported)));
}
}
}

@ParameterizedTest
@ValueSource(strings = {"IF x > 0 SELECT 1 WHERE", "IF x > 0 SELECT * FROM"})
void malformedIfBodyRecordsOneErrorWithoutPublishingPartialIf(String sql) throws Exception {
for (boolean unsupported : new boolean[] {false, true}) {
for (String prefix : new String[] {"", "SELECT 0;"}) {
CCJSqlParser parser = CCJSqlParserUtil.newParser(prefix + sql + ";SELECT 2;")
.withUnsupportedStatements(unsupported).withErrorRecovery(true);
Statements statements = parser.Statements();
int index = prefix.isEmpty() ? 0 : 1;
assertEquals(index + 2, statements.size());
assertNull(statements.get(index));
assertEquals("SELECT 2",
assertInstanceOf(Select.class, statements.get(index + 1)).toString());
assertEquals(1, parser.getParseErrors().size());
}
}
}
}
Loading
Loading