Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: migrate oracle splitter to omni (#20297)
rebelice90518e976205
diff --git a/backend/plugin/db/oracle/oracle_test.go b/backend/plugin/db/oracle/oracle_test.goindex 70f674f133..f03a709849 100644--- a/backend/plugin/db/oracle/oracle_test.go+++ b/backend/plugin/db/oracle/oracle_test.go@@ -5,6 +5,7 @@ import ("github.com/stretchr/testify/require"+ "github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/plugin/parser/plsql")@@ -34,3 +35,46 @@ func TestParseVersion(t *testing.T) {require.Equal(t, test.Second, v.Second)}}++func TestOracleSplitKeepsTrailingFragmentForDatabaseExecution(t *testing.T) {+ commands, err := plsql.SplitSQL("DROP TABLESPACE xxx; CASCADE")+ require.NoError(t, err)+ commands = base.FilterEmptyStatements(commands)+ require.Len(t, commands, 2)+ require.Equal(t, "DROP TABLESPACE xxx", commands[0].Text)+ require.Equal(t, " CASCADE", commands[1].Text)+}++func TestOracleSplitAllowsParserUnsupportedDDL(t *testing.T) {+ statement := `SET DEFINE OFF+CREATE VECTOR INDEX vec_idx ON docs (embedding);+CREATE JSON RELATIONAL DUALITY VIEW emp_dv AS SELECT employee_id FROM employees;+CREATE INDEX IDX_SALES_MONTH_YEAR ON SALES_DATA(EXTRACT(YEAR FROM SALE_DATE), EXTRACT(MONTH FROM SALE_DATE));+CREATE SEQUENCE order_seq START WITH 1 INCREMENT BY 1;+CREATE TABLE employees (salary NUMBER CHECK (salary > 0 OR salary IS NULL));+CREATE PACKAGE pkg IS+ PROCEDURE p;+END pkg;+CREATE PACKAGE BODY pkg IS+ PROCEDURE p IS+ BEGIN+ NULL;+ END p;+END pkg;+CREATE FUNCTION calc_bonus(p_start_date DATE)+RETURN DATE+IS+ v_current_date DATE := p_start_date;+BEGIN+ RETURN v_current_date;+END calc_bonus;+CREATE PROCEDURE update_salary(p_employee_id NUMBER)+IS+BEGIN+ UPDATE employees SET salary = salary + 1 WHERE id = p_employee_id;+END update_salary;`+ commands, err := plsql.SplitSQL(statement)+ require.NoError(t, err)+ commands = base.FilterEmptyStatements(commands)+ require.Len(t, commands, 9)+}diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.goindex 58a0a9ef0a..749394705b 100644--- a/backend/plugin/parser/plsql/split.go+++ b/backend/plugin/parser/plsql/split.go@@ -2,6 +2,7 @@ package plsqlimport ("github.com/antlr4-go/antlr/v4"+ oracleparser "github.com/bytebase/omni/oracle/parser"parser "github.com/bytebase/parser/plsql""github.com/bytebase/bytebase/backend/common"@@ -16,10 +17,8 @@ func init() {// consumeTrailingSemicolon walks forward from stopIdx through hidden-channel// tokens (whitespace, comments) looking for a trailing ';' that belongs to the// statement ending at stopIdx. Returns the index of the ';' if found before any-// default-channel token, otherwise stopIdx (no consumption). Used by both-// SplitSQL and ParsePLSQL to keep the two iterations' separator handling in-// lockstep — divergence between them is what produced BYT-9367's secondary-// AST-classification leak.+// default-channel token, otherwise stopIdx (no consumption). ParsePLSQL uses+// this to avoid BYT-9367's secondary AST-classification leak.func consumeTrailingSemicolon(allTokens []antlr.Token, stopIdx int) int {for nextIdx := stopIdx + 1; nextIdx < len(allTokens); nextIdx++ {next := allTokens[nextIdx]@@ -34,106 +33,47 @@ func consumeTrailingSemicolon(allTokens []antlr.Token, stopIdx int) int {}// SplitSQL splits the given SQL statement into multiple SQL statements.+//+// It uses omni's lexical Oracle splitter, which handles strings, comments,+// SQL*Plus separators, and PL/SQL blocks without requiring valid SQL.func SplitSQL(statement string) ([]base.Statement, error) {- tree, stream, err := ParsePLSQLForStringsManipulation(statement)- if err != nil {- return nil, err- }- if tree == nil {- return nil, nil- }- tokens, ok := stream.(*antlr.CommonTokenStream)- if !ok {- return nil, nil- }+ segments := oracleparser.Split(statement)- byteOffsetStart := 0- prevStopTokenIndex := -1- var result []base.Statement- for _, item := range tree.GetChildren() {- // Skip past sql_plus_command (like "/" block terminator) to prevent it from being- // included in the next statement's leadingContent.- if sqlPlusCmd, ok := item.(parser.ISql_plus_commandContext); ok {- // Calculate the leading whitespace/comments before this sql_plus_command- leadingContent := ""- if startTokenIndex := sqlPlusCmd.GetStart().GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {- leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(sqlPlusCmd.GetStart().GetTokenIndex()-1))- }- // Skip past both the leading content and the command itself- cmdText := tokens.GetTextFromTokens(sqlPlusCmd.GetStart(), sqlPlusCmd.GetStop())- byteOffsetStart += len(leadingContent) + len(cmdText)- prevStopTokenIndex = sqlPlusCmd.GetStop().GetTokenIndex()+ result := make([]base.Statement, 0, len(segments))+ for _, seg := range segments {+ if seg.Kind == oracleparser.SegmentSQLPlusCommand {continue}+ byteEnd := seg.ByteStart + trimOracleSegmentTrailingHidden(seg.Text)+ text := statement[seg.ByteStart:byteEnd]+ result = append(result, base.Statement{+ Text: text,+ Start: ByteOffsetToRunePosition(statement, seg.ByteStart),+ End: ByteOffsetToRunePosition(statement, byteEnd),+ Empty: seg.Empty(),+ Range: &storepb.Range{+ Start: int32(seg.ByteStart),+ End: int32(byteEnd),+ },+ })+ }+ return result, nil+}- if stmt, ok := item.(parser.IUnit_statementContext); ok {- text := ""- var lastToken antlr.Token-- // Calculate the leading whitespace/comments before this statement- leadingContent := ""- if startTokenIndex := stmt.GetStart().GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {- leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(stmt.GetStart().GetTokenIndex()-1))- }-- // The go-ora driver requires semicolon for anonymous blocks/procedures/functions,- // but does NOT support semicolon for other statements (CREATE TABLE, SELECT, etc.).- stopTokenIndex := stmt.GetStop().GetTokenIndex()- if needSemicolon(stmt) {- // For procedures/functions/anonymous blocks: include semicolon if present, add if missing- lastToken = tokens.Get(stopTokenIndex)- text = leadingContent + tokens.GetTextFromTokens(stmt.GetStart(), lastToken)- if lastToken.GetTokenType() != parser.PlSqlParserSEMICOLON {- text += ";"- }- } else {- // For regular statements: EXCLUDE the semicolon (go-ora doesn't support it)- if stmt.GetStop().GetTokenType() == parser.PlSqlParserSEMICOLON {- stopTokenIndex--- }- lastToken = tokens.Get(stopTokenIndex)- text = leadingContent + tokens.GetTextFromTokens(stmt.GetStart(), lastToken)- }-- // Calculate byte offsets using lastToken (which includes semicolon if present)- // byteOffsetStart is where the previous statement ended (including any leading whitespace)- tokenByteOffset := byteOffsetStart + len(leadingContent)- byteOffsetEnd := tokenByteOffset + len(tokens.GetTextFromTokens(stmt.GetStart(), lastToken))-- // Calculate start position based on byteOffsetStart (including leading whitespace)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffsetStart)-- result = append(result, base.Statement{- Text: text,- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(lastToken.GetLine()),- int32(lastToken.GetColumn()),- lastToken.GetText(),- ),- Empty: base.IsEmpty(tokens.GetAllTokens()[stmt.GetStart().GetTokenIndex():stmt.GetStop().GetTokenIndex()+1], parser.PlSqlParserSEMICOLON),- Range: &storepb.Range{- Start: int32(byteOffsetStart),- End: int32(byteOffsetEnd),- },- })- byteOffsetStart = byteOffsetEnd- // If a trailing ';' was consumed across hidden tokens, advance- // byteOffsetStart by the BYTE length of the consumed span — len()- // is bytes; ANTLR token indices are runes, so multi-byte UTF-8 in- // hidden tokens (e.g., a non-ASCII comment) would diverge.- stopIdx := stmt.GetStop().GetTokenIndex()- allTokens := tokens.GetAllTokens()- prevStopTokenIndex = consumeTrailingSemicolon(allTokens, stopIdx)- if prevStopTokenIndex > stopIdx {- byteOffsetStart += len(tokens.GetTextFromTokens(allTokens[stopIdx+1], allTokens[prevStopTokenIndex]))- }+func trimOracleSegmentTrailingHidden(text string) int {+ lexer := oracleparser.NewLexer(text)+ lastTokenEnd := 0+ for {+ token := lexer.NextToken()+ if token.Type == 0 {+ break}+ lastTokenEnd = token.End}- return result, nil+ if lexer.Err != nil || lastTokenEnd == 0 {+ return len(text)+ }+ return lastTokenEnd}func SplitSQLForCompletion(statement string) ([]base.Statement, error) {@@ -193,20 +133,3 @@ func isCallStatement(item antlr.Tree) bool {// BYT-8268: Changed from Call_statement to Sql_call_statementreturn unitStmt.Sql_call_statement() != nil}--// needSemicolon returns true if the given statement needs a semicolon.-// The go-ora driver requires semicolon for anonymous block and create procedure/function/package/trigger type of statements,-// but does not support semicolon for other statements.-func needSemicolon(stmt parser.IUnit_statementContext) bool {- switch {- case stmt.Anonymous_block() != nil,- stmt.Create_procedure_body() != nil,- stmt.Create_function_body() != nil,- stmt.Create_package() != nil,- stmt.Create_package_body() != nil,- stmt.Create_trigger() != nil:- return true- default:- return false- }-}diff --git a/backend/plugin/parser/plsql/split_test.go b/backend/plugin/parser/plsql/split_test.goindex 91d5c25126..b94a83b15a 100644--- a/backend/plugin/parser/plsql/split_test.go+++ b/backend/plugin/parser/plsql/split_test.go@@ -3,6 +3,8 @@ package plsqlimport ("testing"+ "github.com/stretchr/testify/require"+"github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -11,3 +13,56 @@ func TestPLSQLSplitSQL(t *testing.T) {SplitFunc: SplitSQL,})}++func TestPLSQLSplitSQLStoredUnitWithoutSlashSeparator(t *testing.T) {+ statement := `CREATE FUNCTION calc_bonus(p_start_date DATE)+RETURN DATE+IS+ v_current_date DATE := p_start_date;+BEGIN+ RETURN v_current_date;+END calc_bonus;+CREATE PROCEDURE update_salary(p_employee_id NUMBER)+IS+BEGIN+ UPDATE employees SET salary = salary + 1 WHERE id = p_employee_id;+END update_salary;+CREATE TABLE t(id NUMBER);`++ statements, err := SplitSQL(statement)+ require.NoError(t, err)+ statements = base.FilterEmptyStatements(statements)++ require.Len(t, statements, 3)+ require.Equal(t, `CREATE FUNCTION calc_bonus(p_start_date DATE)+RETURN DATE+IS+ v_current_date DATE := p_start_date;+BEGIN+ RETURN v_current_date;+END calc_bonus;`, statements[0].Text)+ require.Equal(t, `+CREATE PROCEDURE update_salary(p_employee_id NUMBER)+IS+BEGIN+ UPDATE employees SET salary = salary + 1 WHERE id = p_employee_id;+END update_salary;`, statements[1].Text)+ require.Equal(t, `+CREATE TABLE t(id NUMBER)`, statements[2].Text)+}++func TestPLSQLSplitSQLPreservesSQLSetStatements(t *testing.T) {+ statement := `SET DEFINE OFF+SET TRANSACTION READ ONLY;+SET ROLE app_role;+SELECT 1 FROM dual;`++ statements, err := SplitSQL(statement)+ require.NoError(t, err)+ statements = base.FilterEmptyStatements(statements)++ require.Len(t, statements, 3)+ require.Equal(t, "SET TRANSACTION READ ONLY", statements[0].Text)+ require.Equal(t, "\nSET ROLE app_role", statements[1].Text)+ require.Equal(t, "\nSELECT 1 FROM dual", statements[2].Text)+}diff --git a/backend/plugin/parser/plsql/test-data/test_split.yaml b/backend/plugin/parser/plsql/test-data/test_split.yamlindex 4c8fb73d3f..f05b8c93ba 100644--- a/backend/plugin/parser/plsql/test-data/test_split.yaml+++ b/backend/plugin/parser/plsql/test-data/test_split.yaml@@ -25,6 +25,148 @@start: 17end: 34empty: false+- description: modern Oracle DDL supported by omni splitter+ input: "CREATE VECTOR INDEX vec_idx ON docs (embedding);\nCREATE JSON RELATIONAL DUALITY VIEW emp_dv AS SELECT employee_id FROM employees;"+ result:+ - text: CREATE VECTOR INDEX vec_idx ON docs (embedding)+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 48+ range:+ start: 0+ end: 47+ empty: false+ - text: "\nCREATE JSON RELATIONAL DUALITY VIEW emp_dv AS SELECT employee_id FROM employees"+ baseline: 0+ start:+ line: 1+ column: 49+ end:+ line: 2+ column: 80+ range:+ start: 48+ end: 128+ empty: false+- description: SQLPlus line commands are skipped+ input: "SET DEFINE OFF\nPROMPT creating table\nSELECT 1 FROM dual;\nSPOOL install.log\nSELECT 2 FROM dual;\nSPOOL OFF"+ result:+ - text: SELECT 1 FROM dual+ baseline: 2+ start:+ line: 3+ column: 1+ end:+ line: 3+ column: 19+ range:+ start: 37+ end: 55+ empty: false+ - text: SELECT 2 FROM dual+ baseline: 4+ start:+ line: 5+ column: 1+ end:+ line: 5+ column: 19+ range:+ start: 75+ end: 93+ empty: false+- description: SQLPlus script commands are skipped+ input: "@preflight.sql\n@@nested/install.sql arg1 arg2\nSTART post.sql\nSELECT 1 FROM dual;\nEXIT SUCCESS"+ result:+ - text: SELECT 1 FROM dual+ baseline: 3+ start:+ line: 4+ column: 1+ end:+ line: 4+ column: 19+ range:+ start: 61+ end: 79+ empty: false+- description: SQLPlus command words inside SQL are retained+ input: "SELECT 'SET DEFINE OFF' AS txt FROM dual; SELECT prompt FROM t;"+ result:+ - text: SELECT 'SET DEFINE OFF' AS txt FROM dual+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 41+ range:+ start: 0+ end: 40+ empty: false+ - text: " SELECT prompt FROM t"+ baseline: 0+ start:+ line: 1+ column: 42+ end:+ line: 1+ column: 63+ range:+ start: 41+ end: 62+ empty: false+- description: package spec and body without slash separators+ input: |-+ CREATE PACKAGE pkg IS+ PROCEDURE p;+ END pkg;+ CREATE PACKAGE BODY pkg IS+ PROCEDURE p IS+ BEGIN+ NULL;+ END;+ END pkg;+ result:+ - text: |-+ CREATE PACKAGE pkg IS+ PROCEDURE p;+ END pkg;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 9+ range:+ start: 0+ end: 45+ empty: false+ - text: |-++ CREATE PACKAGE BODY pkg IS+ PROCEDURE p IS+ BEGIN+ NULL;+ END;+ END pkg;+ baseline: 2+ start:+ line: 3+ column: 9+ end:+ line: 9+ column: 9+ range:+ start: 45+ end: 123+ empty: false- description: multiple statements with newlinesinput: "\n\t\t\t\tselect * from t;\n\t\t\t\tcreate table table$1 (id int)\n\t\t\t"result:@@ -232,12 +374,48 @@start: 0end: 58empty: false-- description: DROP TABLESPACE CASCADE alone should error+- description: DROP TABLESPACE CASCADE alone is split lexicallyinput: DROP TABLESPACE xxx CASCADE- error: "Syntax error at line 1:28 \nrelated text: DROP TABLESPACE xxx CASCADE;"-- description: DROP TABLESPACE then CASCADE should error+ result:+ - text: DROP TABLESPACE xxx CASCADE+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 28+ range:+ start: 0+ end: 27+ empty: false+- description: DROP TABLESPACE then CASCADE is split lexicallyinput: DROP TABLESPACE xxx; CASCADE- error: "Syntax error at line 1:29 \nrelated text: DROP TABLESPACE xxx; CASCADE;"+ result:+ - text: DROP TABLESPACE xxx+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 20+ range:+ start: 0+ end: 19+ empty: false+ - text: " CASCADE"+ baseline: 0+ start:+ line: 1+ column: 21+ end:+ line: 1+ column: 29+ range:+ start: 20+ end: 28+ empty: false- description: CREATE TABLE with ROW STORE COMPRESS ADVANCEDinput: |-CREATE TABLE DATA.SMY_LAO_ACCOUNT_BLK_TXN_MTH (