Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

chore: migrate oracle splitter to omni (#20297)

rebelice90518e976205
diff --git a/backend/plugin/db/oracle/oracle_test.go b/backend/plugin/db/oracle/oracle_test.go
index 70f674f133..f03a709849 100644
--- a/backend/plugin/db/oracle/oracle_test.go
+++ b/backend/plugin/db/oracle/oracle_test.go
@@ -5,6 +5,7 @@ import (
"github.com/stretchr/testify/require"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/plugin/parser/plsql"
)
@@ -34,3 +35,46 @@ func TestParseVersion(t *testing.T) {
require.Equal(t, test.Second, v.Second)
}
}
+
+func TestOracleSplitKeepsTrailingFragmentForDatabaseExecution(t *testing.T) {
+ commands, err := plsql.SplitSQL("DROP TABLESPACE xxx; CASCADE")
+ require.NoError(t, err)
+ commands = base.FilterEmptyStatements(commands)
+ require.Len(t, commands, 2)
+ require.Equal(t, "DROP TABLESPACE xxx", commands[0].Text)
+ require.Equal(t, " CASCADE", commands[1].Text)
+}
+
+func TestOracleSplitAllowsParserUnsupportedDDL(t *testing.T) {
+ statement := `SET DEFINE OFF
+CREATE VECTOR INDEX vec_idx ON docs (embedding);
+CREATE JSON RELATIONAL DUALITY VIEW emp_dv AS SELECT employee_id FROM employees;
+CREATE INDEX IDX_SALES_MONTH_YEAR ON SALES_DATA(EXTRACT(YEAR FROM SALE_DATE), EXTRACT(MONTH FROM SALE_DATE));
+CREATE SEQUENCE order_seq START WITH 1 INCREMENT BY 1;
+CREATE TABLE employees (salary NUMBER CHECK (salary > 0 OR salary IS NULL));
+CREATE PACKAGE pkg IS
+ PROCEDURE p;
+END pkg;
+CREATE PACKAGE BODY pkg IS
+ PROCEDURE p IS
+ BEGIN
+ NULL;
+ END p;
+END pkg;
+CREATE FUNCTION calc_bonus(p_start_date DATE)
+RETURN DATE
+IS
+ v_current_date DATE := p_start_date;
+BEGIN
+ RETURN v_current_date;
+END calc_bonus;
+CREATE PROCEDURE update_salary(p_employee_id NUMBER)
+IS
+BEGIN
+ UPDATE employees SET salary = salary + 1 WHERE id = p_employee_id;
+END update_salary;`
+ commands, err := plsql.SplitSQL(statement)
+ require.NoError(t, err)
+ commands = base.FilterEmptyStatements(commands)
+ require.Len(t, commands, 9)
+}
diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.go
index 58a0a9ef0a..749394705b 100644
--- a/backend/plugin/parser/plsql/split.go
+++ b/backend/plugin/parser/plsql/split.go
@@ -2,6 +2,7 @@ package plsql
import (
"github.com/antlr4-go/antlr/v4"
+ oracleparser "github.com/bytebase/omni/oracle/parser"
parser "github.com/bytebase/parser/plsql"
"github.com/bytebase/bytebase/backend/common"
@@ -16,10 +17,8 @@ func init() {
// consumeTrailingSemicolon walks forward from stopIdx through hidden-channel
// tokens (whitespace, comments) looking for a trailing ';' that belongs to the
// statement ending at stopIdx. Returns the index of the ';' if found before any
-// default-channel token, otherwise stopIdx (no consumption). Used by both
-// SplitSQL and ParsePLSQL to keep the two iterations' separator handling in
-// lockstep — divergence between them is what produced BYT-9367's secondary
-// AST-classification leak.
+// default-channel token, otherwise stopIdx (no consumption). ParsePLSQL uses
+// this to avoid BYT-9367's secondary AST-classification leak.
func consumeTrailingSemicolon(allTokens []antlr.Token, stopIdx int) int {
for nextIdx := stopIdx + 1; nextIdx < len(allTokens); nextIdx++ {
next := allTokens[nextIdx]
@@ -34,106 +33,47 @@ func consumeTrailingSemicolon(allTokens []antlr.Token, stopIdx int) int {
}
// SplitSQL splits the given SQL statement into multiple SQL statements.
+//
+// It uses omni's lexical Oracle splitter, which handles strings, comments,
+// SQL*Plus separators, and PL/SQL blocks without requiring valid SQL.
func SplitSQL(statement string) ([]base.Statement, error) {
- tree, stream, err := ParsePLSQLForStringsManipulation(statement)
- if err != nil {
- return nil, err
- }
- if tree == nil {
- return nil, nil
- }
- tokens, ok := stream.(*antlr.CommonTokenStream)
- if !ok {
- return nil, nil
- }
+ segments := oracleparser.Split(statement)
- byteOffsetStart := 0
- prevStopTokenIndex := -1
- var result []base.Statement
- for _, item := range tree.GetChildren() {
- // Skip past sql_plus_command (like "/" block terminator) to prevent it from being
- // included in the next statement's leadingContent.
- if sqlPlusCmd, ok := item.(parser.ISql_plus_commandContext); ok {
- // Calculate the leading whitespace/comments before this sql_plus_command
- leadingContent := ""
- if startTokenIndex := sqlPlusCmd.GetStart().GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {
- leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(sqlPlusCmd.GetStart().GetTokenIndex()-1))
- }
- // Skip past both the leading content and the command itself
- cmdText := tokens.GetTextFromTokens(sqlPlusCmd.GetStart(), sqlPlusCmd.GetStop())
- byteOffsetStart += len(leadingContent) + len(cmdText)
- prevStopTokenIndex = sqlPlusCmd.GetStop().GetTokenIndex()
+ result := make([]base.Statement, 0, len(segments))
+ for _, seg := range segments {
+ if seg.Kind == oracleparser.SegmentSQLPlusCommand {
continue
}
+ byteEnd := seg.ByteStart + trimOracleSegmentTrailingHidden(seg.Text)
+ text := statement[seg.ByteStart:byteEnd]
+ result = append(result, base.Statement{
+ Text: text,
+ Start: ByteOffsetToRunePosition(statement, seg.ByteStart),
+ End: ByteOffsetToRunePosition(statement, byteEnd),
+ Empty: seg.Empty(),
+ Range: &storepb.Range{
+ Start: int32(seg.ByteStart),
+ End: int32(byteEnd),
+ },
+ })
+ }
+ return result, nil
+}
- if stmt, ok := item.(parser.IUnit_statementContext); ok {
- text := ""
- var lastToken antlr.Token
-
- // Calculate the leading whitespace/comments before this statement
- leadingContent := ""
- if startTokenIndex := stmt.GetStart().GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {
- leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(stmt.GetStart().GetTokenIndex()-1))
- }
-
- // The go-ora driver requires semicolon for anonymous blocks/procedures/functions,
- // but does NOT support semicolon for other statements (CREATE TABLE, SELECT, etc.).
- stopTokenIndex := stmt.GetStop().GetTokenIndex()
- if needSemicolon(stmt) {
- // For procedures/functions/anonymous blocks: include semicolon if present, add if missing
- lastToken = tokens.Get(stopTokenIndex)
- text = leadingContent + tokens.GetTextFromTokens(stmt.GetStart(), lastToken)
- if lastToken.GetTokenType() != parser.PlSqlParserSEMICOLON {
- text += ";"
- }
- } else {
- // For regular statements: EXCLUDE the semicolon (go-ora doesn't support it)
- if stmt.GetStop().GetTokenType() == parser.PlSqlParserSEMICOLON {
- stopTokenIndex--
- }
- lastToken = tokens.Get(stopTokenIndex)
- text = leadingContent + tokens.GetTextFromTokens(stmt.GetStart(), lastToken)
- }
-
- // Calculate byte offsets using lastToken (which includes semicolon if present)
- // byteOffsetStart is where the previous statement ended (including any leading whitespace)
- tokenByteOffset := byteOffsetStart + len(leadingContent)
- byteOffsetEnd := tokenByteOffset + len(tokens.GetTextFromTokens(stmt.GetStart(), lastToken))
-
- // Calculate start position based on byteOffsetStart (including leading whitespace)
- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffsetStart)
-
- result = append(result, base.Statement{
- Text: text,
- Start: &storepb.Position{
- Line: int32(startLine + 1),
- Column: int32(startColumn + 1),
- },
- End: common.ConvertANTLRTokenToExclusiveEndPosition(
- int32(lastToken.GetLine()),
- int32(lastToken.GetColumn()),
- lastToken.GetText(),
- ),
- Empty: base.IsEmpty(tokens.GetAllTokens()[stmt.GetStart().GetTokenIndex():stmt.GetStop().GetTokenIndex()+1], parser.PlSqlParserSEMICOLON),
- Range: &storepb.Range{
- Start: int32(byteOffsetStart),
- End: int32(byteOffsetEnd),
- },
- })
- byteOffsetStart = byteOffsetEnd
- // If a trailing ';' was consumed across hidden tokens, advance
- // byteOffsetStart by the BYTE length of the consumed span — len()
- // is bytes; ANTLR token indices are runes, so multi-byte UTF-8 in
- // hidden tokens (e.g., a non-ASCII comment) would diverge.
- stopIdx := stmt.GetStop().GetTokenIndex()
- allTokens := tokens.GetAllTokens()
- prevStopTokenIndex = consumeTrailingSemicolon(allTokens, stopIdx)
- if prevStopTokenIndex > stopIdx {
- byteOffsetStart += len(tokens.GetTextFromTokens(allTokens[stopIdx+1], allTokens[prevStopTokenIndex]))
- }
+func trimOracleSegmentTrailingHidden(text string) int {
+ lexer := oracleparser.NewLexer(text)
+ lastTokenEnd := 0
+ for {
+ token := lexer.NextToken()
+ if token.Type == 0 {
+ break
}
+ lastTokenEnd = token.End
}
- return result, nil
+ if lexer.Err != nil || lastTokenEnd == 0 {
+ return len(text)
+ }
+ return lastTokenEnd
}
func SplitSQLForCompletion(statement string) ([]base.Statement, error) {
@@ -193,20 +133,3 @@ func isCallStatement(item antlr.Tree) bool {
// BYT-8268: Changed from Call_statement to Sql_call_statement
return unitStmt.Sql_call_statement() != nil
}
-
-// needSemicolon returns true if the given statement needs a semicolon.
-// The go-ora driver requires semicolon for anonymous block and create procedure/function/package/trigger type of statements,
-// but does not support semicolon for other statements.
-func needSemicolon(stmt parser.IUnit_statementContext) bool {
- switch {
- case stmt.Anonymous_block() != nil,
- stmt.Create_procedure_body() != nil,
- stmt.Create_function_body() != nil,
- stmt.Create_package() != nil,
- stmt.Create_package_body() != nil,
- stmt.Create_trigger() != nil:
- return true
- default:
- return false
- }
-}
diff --git a/backend/plugin/parser/plsql/split_test.go b/backend/plugin/parser/plsql/split_test.go
index 91d5c25126..b94a83b15a 100644
--- a/backend/plugin/parser/plsql/split_test.go
+++ b/backend/plugin/parser/plsql/split_test.go
@@ -3,6 +3,8 @@ package plsql
import (
"testing"
+ "github.com/stretchr/testify/require"
+
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -11,3 +13,56 @@ func TestPLSQLSplitSQL(t *testing.T) {
SplitFunc: SplitSQL,
})
}
+
+func TestPLSQLSplitSQLStoredUnitWithoutSlashSeparator(t *testing.T) {
+ statement := `CREATE FUNCTION calc_bonus(p_start_date DATE)
+RETURN DATE
+IS
+ v_current_date DATE := p_start_date;
+BEGIN
+ RETURN v_current_date;
+END calc_bonus;
+CREATE PROCEDURE update_salary(p_employee_id NUMBER)
+IS
+BEGIN
+ UPDATE employees SET salary = salary + 1 WHERE id = p_employee_id;
+END update_salary;
+CREATE TABLE t(id NUMBER);`
+
+ statements, err := SplitSQL(statement)
+ require.NoError(t, err)
+ statements = base.FilterEmptyStatements(statements)
+
+ require.Len(t, statements, 3)
+ require.Equal(t, `CREATE FUNCTION calc_bonus(p_start_date DATE)
+RETURN DATE
+IS
+ v_current_date DATE := p_start_date;
+BEGIN
+ RETURN v_current_date;
+END calc_bonus;`, statements[0].Text)
+ require.Equal(t, `
+CREATE PROCEDURE update_salary(p_employee_id NUMBER)
+IS
+BEGIN
+ UPDATE employees SET salary = salary + 1 WHERE id = p_employee_id;
+END update_salary;`, statements[1].Text)
+ require.Equal(t, `
+CREATE TABLE t(id NUMBER)`, statements[2].Text)
+}
+
+func TestPLSQLSplitSQLPreservesSQLSetStatements(t *testing.T) {
+ statement := `SET DEFINE OFF
+SET TRANSACTION READ ONLY;
+SET ROLE app_role;
+SELECT 1 FROM dual;`
+
+ statements, err := SplitSQL(statement)
+ require.NoError(t, err)
+ statements = base.FilterEmptyStatements(statements)
+
+ require.Len(t, statements, 3)
+ require.Equal(t, "SET TRANSACTION READ ONLY", statements[0].Text)
+ require.Equal(t, "\nSET ROLE app_role", statements[1].Text)
+ require.Equal(t, "\nSELECT 1 FROM dual", statements[2].Text)
+}
diff --git a/backend/plugin/parser/plsql/test-data/test_split.yaml b/backend/plugin/parser/plsql/test-data/test_split.yaml
index 4c8fb73d3f..f05b8c93ba 100644
--- a/backend/plugin/parser/plsql/test-data/test_split.yaml
+++ b/backend/plugin/parser/plsql/test-data/test_split.yaml
@@ -25,6 +25,148 @@
start: 17
end: 34
empty: false
+- description: modern Oracle DDL supported by omni splitter
+ input: "CREATE VECTOR INDEX vec_idx ON docs (embedding);\nCREATE JSON RELATIONAL DUALITY VIEW emp_dv AS SELECT employee_id FROM employees;"
+ result:
+ - text: CREATE VECTOR INDEX vec_idx ON docs (embedding)
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 48
+ range:
+ start: 0
+ end: 47
+ empty: false
+ - text: "\nCREATE JSON RELATIONAL DUALITY VIEW emp_dv AS SELECT employee_id FROM employees"
+ baseline: 0
+ start:
+ line: 1
+ column: 49
+ end:
+ line: 2
+ column: 80
+ range:
+ start: 48
+ end: 128
+ empty: false
+- description: SQLPlus line commands are skipped
+ input: "SET DEFINE OFF\nPROMPT creating table\nSELECT 1 FROM dual;\nSPOOL install.log\nSELECT 2 FROM dual;\nSPOOL OFF"
+ result:
+ - text: SELECT 1 FROM dual
+ baseline: 2
+ start:
+ line: 3
+ column: 1
+ end:
+ line: 3
+ column: 19
+ range:
+ start: 37
+ end: 55
+ empty: false
+ - text: SELECT 2 FROM dual
+ baseline: 4
+ start:
+ line: 5
+ column: 1
+ end:
+ line: 5
+ column: 19
+ range:
+ start: 75
+ end: 93
+ empty: false
+- description: SQLPlus script commands are skipped
+ input: "@preflight.sql\n@@nested/install.sql arg1 arg2\nSTART post.sql\nSELECT 1 FROM dual;\nEXIT SUCCESS"
+ result:
+ - text: SELECT 1 FROM dual
+ baseline: 3
+ start:
+ line: 4
+ column: 1
+ end:
+ line: 4
+ column: 19
+ range:
+ start: 61
+ end: 79
+ empty: false
+- description: SQLPlus command words inside SQL are retained
+ input: "SELECT 'SET DEFINE OFF' AS txt FROM dual; SELECT prompt FROM t;"
+ result:
+ - text: SELECT 'SET DEFINE OFF' AS txt FROM dual
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 41
+ range:
+ start: 0
+ end: 40
+ empty: false
+ - text: " SELECT prompt FROM t"
+ baseline: 0
+ start:
+ line: 1
+ column: 42
+ end:
+ line: 1
+ column: 63
+ range:
+ start: 41
+ end: 62
+ empty: false
+- description: package spec and body without slash separators
+ input: |-
+ CREATE PACKAGE pkg IS
+ PROCEDURE p;
+ END pkg;
+ CREATE PACKAGE BODY pkg IS
+ PROCEDURE p IS
+ BEGIN
+ NULL;
+ END;
+ END pkg;
+ result:
+ - text: |-
+ CREATE PACKAGE pkg IS
+ PROCEDURE p;
+ END pkg;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 9
+ range:
+ start: 0
+ end: 45
+ empty: false
+ - text: |-
+
+ CREATE PACKAGE BODY pkg IS
+ PROCEDURE p IS
+ BEGIN
+ NULL;
+ END;
+ END pkg;
+ baseline: 2
+ start:
+ line: 3
+ column: 9
+ end:
+ line: 9
+ column: 9
+ range:
+ start: 45
+ end: 123
+ empty: false
- description: multiple statements with newlines
input: "\n\t\t\t\tselect * from t;\n\t\t\t\tcreate table table$1 (id int)\n\t\t\t"
result:
@@ -232,12 +374,48 @@
start: 0
end: 58
empty: false
-- description: DROP TABLESPACE CASCADE alone should error
+- description: DROP TABLESPACE CASCADE alone is split lexically
input: DROP TABLESPACE xxx CASCADE
- error: "Syntax error at line 1:28 \nrelated text: DROP TABLESPACE xxx CASCADE;"
-- description: DROP TABLESPACE then CASCADE should error
+ result:
+ - text: DROP TABLESPACE xxx CASCADE
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 28
+ range:
+ start: 0
+ end: 27
+ empty: false
+- description: DROP TABLESPACE then CASCADE is split lexically
input: DROP TABLESPACE xxx; CASCADE
- error: "Syntax error at line 1:29 \nrelated text: DROP TABLESPACE xxx; CASCADE;"
+ result:
+ - text: DROP TABLESPACE xxx
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 20
+ range:
+ start: 0
+ end: 19
+ empty: false
+ - text: " CASCADE"
+ baseline: 0
+ start:
+ line: 1
+ column: 21
+ end:
+ line: 1
+ column: 29
+ range:
+ start: 20
+ end: 28
+ empty: false
- description: CREATE TABLE with ROW STORE COMPRESS ADVANCED
input: |-
CREATE TABLE DATA.SMY_LAO_ACCOUNT_BLK_TXN_MTH (