Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore(plsql): add omni-first parse dispatcher (#20349)
rebelice3556a1d7d167
diff --git a/backend/plugin/parser/base/base.go b/backend/plugin/parser/base/base.goindex 9383d03b84..b706316421 100644--- a/backend/plugin/parser/base/base.go+++ b/backend/plugin/parser/base/base.go@@ -37,8 +37,12 @@ func (l *ParseErrorListener) SyntaxError(_ antlr.Recognizer, token any, line, co// Get 0-based line offset from StartPosition (1-based) for calculationslineOffset := int32(0)+ columnOffset := int32(0)if l.StartPosition != nil {lineOffset = l.StartPosition.Line - 1+ if line == 1 && l.StartPosition.Column > 0 {+ columnOffset = l.StartPosition.Column - 1+ }}errMessage := ""@@ -57,7 +61,7 @@ func (l *ParseErrorListener) SyntaxError(_ antlr.Recognizer, token any, line, co// ANTLR provides 1-based line and 0-based column// Store as 1-based line and 1-based column in PositionposLine := int32(line) + lineOffset- posColumn := int32(column + 1)+ posColumn := int32(column+1) + columnOffsetl.Err = &SyntaxError{Position: &storepb.Position{Line: posLine,diff --git a/backend/plugin/parser/plsql/omni.go b/backend/plugin/parser/plsql/omni.goindex 3968e3da6e..af7fe4eb2c 100644--- a/backend/plugin/parser/plsql/omni.go+++ b/backend/plugin/parser/plsql/omni.go@@ -3,14 +3,18 @@ package plsqlimport ("unicode/utf8"+ "github.com/antlr4-go/antlr/v4""github.com/bytebase/omni/oracle/ast"oracleparser "github.com/bytebase/omni/oracle/parser"+ antlrparser "github.com/bytebase/parser/plsql"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")// OmniAST wraps an omni AST node and implements the base.AST interface.+// During migration, it also implements AntlrASTProvider so that callers+// still using GetANTLRAST() can fall back to the ANTLR tree.type OmniAST struct {// Node is the omni AST node (e.g. *ast.SelectStmt, *ast.CreateTableStmt).Node ast.Node@@ -18,6 +22,12 @@ type OmniAST struct {Text string// StartPosition is the 1-based position where this statement starts.StartPosition *storepb.Position++ // antlrAST is lazily populated when AsANTLRAST() is called.+ // This field will be removed once all Oracle modules are migrated to omni.+ antlrAST *base.ANTLRAST+ // antlrParsed tracks whether we've attempted the ANTLR parse.+ antlrParsed bool}// ASTStartPosition implements base.AST.@@ -25,6 +35,41 @@ func (a *OmniAST) ASTStartPosition() *storepb.Position {return a.StartPosition}+// AsANTLRAST implements base.AntlrASTProvider for backward compatibility.+// It lazily parses the SQL text with the ANTLR parser and caches the result.+func (a *OmniAST) AsANTLRAST() (*base.ANTLRAST, bool) {+ if a.antlrParsed {+ return a.antlrAST, a.antlrAST != nil+ }+ a.antlrParsed = true++ tree, tokens := parseSinglePLSQLLenient(a.Text)+ a.antlrAST = &base.ANTLRAST{+ StartPosition: a.StartPosition,+ Tree: tree,+ Tokens: tokens,+ }+ return a.antlrAST, true+}++// parseSinglePLSQLLenient parses a single Oracle statement without error listeners.+// The omni parser has already validated the SQL; this tree only exists for+// backward-compatible ANTLR consumers during the migration window.+func parseSinglePLSQLLenient(statement string) (antlr.Tree, *antlr.CommonTokenStream) {+ inputStream := antlr.NewInputStream(addSemicolonIfNeeded(statement))+ lexer := antlrparser.NewPlSqlLexer(inputStream)+ stream := antlr.NewCommonTokenStream(lexer, 0)+ p := antlrparser.NewPlSqlParser(stream)+ p.SetVersion12(true)++ lexer.RemoveErrorListeners()+ p.RemoveErrorListeners()+ p.BuildParseTrees = true++ tree := p.Sql_script()+ return tree, stream+}+// ParsePLSQLOmni parses SQL using omni's parser and returns an ast.List.// This is the recommended entry point for new Oracle code that needs omni AST nodes.func ParsePLSQLOmni(sql string) (*ast.List, error) {diff --git a/backend/plugin/parser/plsql/omni_test.go b/backend/plugin/parser/plsql/omni_test.goindex abb917b488..cf4cf5875f 100644--- a/backend/plugin/parser/plsql/omni_test.go+++ b/backend/plugin/parser/plsql/omni_test.go@@ -3,6 +3,7 @@ package plsqlimport ("testing"+ parser "github.com/bytebase/parser/plsql""github.com/stretchr/testify/require""github.com/bytebase/omni/oracle/ast"@@ -52,6 +53,25 @@ func TestOracleOmniASTWrapper(t *testing.T) {require.Nil(t, got)}+func TestOracleOmniASTAsANTLRAST(t *testing.T) {+ start := &storepb.Position{Line: 4, Column: 1}+ omniAST := &OmniAST{+ Node: &ast.SelectStmt{},+ Text: "SELECT * FROM T",+ StartPosition: start,+ }++ antlrAST, ok := base.GetANTLRAST(omniAST)+ require.True(t, ok)+ require.Equal(t, start, antlrAST.StartPosition)+ require.IsType(t, &parser.Sql_scriptContext{}, antlrAST.Tree)+ require.NotNil(t, antlrAST.Tokens)++ antlrASTAgain, ok := base.GetANTLRAST(omniAST)+ require.True(t, ok)+ require.Same(t, antlrAST, antlrASTAgain)+}+func TestOracleByteOffsetToRunePosition(t *testing.T) {position := ByteOffsetToRunePosition("SELECT\n日本 FROM DUAL", len("SELECT\n日"))require.Equal(t, int32(2), position.Line)diff --git a/backend/plugin/parser/plsql/plsql.go b/backend/plugin/parser/plsql/plsql.goindex 0cc11ce1d2..f193401e30 100644--- a/backend/plugin/parser/plsql/plsql.go+++ b/backend/plugin/parser/plsql/plsql.go@@ -7,6 +7,7 @@ import ("strings""github.com/antlr4-go/antlr/v4"+ "github.com/bytebase/omni/oracle/ast"parser "github.com/bytebase/parser/plsql""github.com/pkg/errors"@@ -28,24 +29,50 @@ func parsePLSQLStatements(statement string) ([]base.ParsedStatement, error) {return nil, err}- // Then parse to get ASTs (note: ParsePLSQL adds semicolon internally)- parseResults, err := ParsePLSQL(statement + ";")- if err != nil {- return nil, err- }-- // Combine: Statement provides text/positions, ANTLRAST provides ASTvar result []base.ParsedStatement- astIndex := 0for _, stmt := range stmts {- ps := base.ParsedStatement{- Statement: stmt,+ if stmt.Empty {+ result = append(result, base.ParsedStatement{Statement: stmt})+ continue+ }++ list, omniErr := ParsePLSQLOmni(stmt.Text)+ if omniErr == nil {+ if list == nil || len(list.Items) == 0 {+ result = append(result, base.ParsedStatement{Statement: stmt})+ continue+ }+ for _, node := range list.Items {+ raw, ok := node.(*ast.RawStmt)+ if !ok {+ continue+ }+ result = append(result, base.ParsedStatement{+ Statement: stmt,+ AST: &OmniAST{+ Node: raw.Stmt,+ Text: stmt.Text,+ StartPosition: stmt.Start,+ },+ })+ }+ continue+ }++ parseResults, err := parsePLSQLWithStartPosition(stmt.Text, stmt.Start)+ if err != nil {+ return nil, err+ }+ if len(parseResults) == 0 {+ result = append(result, base.ParsedStatement{Statement: stmt})+ continue}- if !stmt.Empty && astIndex < len(parseResults) {- ps.AST = parseResults[astIndex]- astIndex+++ for _, parseResult := range parseResults {+ result = append(result, base.ParsedStatement{+ Statement: stmt,+ AST: parseResult,+ })}- result = append(result, ps)}return result, nil@@ -88,10 +115,18 @@ func ParseVersion(banner string) (*Version, error) {// It first parses the whole statement to get the AST, then splits by unit_statement// and sql_plus_command nodes, and re-parses each individual statement.func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {+ return parsePLSQLWithBaseLine(sql, 0)+}++func parsePLSQLWithBaseLine(sql string, baseLine int) ([]*base.ANTLRAST, error) {+ return parsePLSQLWithStartPosition(sql, &storepb.Position{Line: int32(baseLine) + 1})+}++func parsePLSQLWithStartPosition(sql string, startPosition *storepb.Position) ([]*base.ANTLRAST, error) {sql = addSemicolonIfNeeded(sql)// First pass: parse the whole statement to get the AST for splitting- tree, tokens, err := parsePLSQLInternal(sql, 0)+ tree, tokens, err := parsePLSQLInternal(sql, startPosition)if err != nil {return nil, err}@@ -134,7 +169,7 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {// stmtBaseLine is where the leading content starts (for re-parsing with correct offsets).// This ensures token positions in the re-parsed AST are correct when combined with SplitSQL's BaseLine.// Formula: first token's line - 1 (convert to 0-based) - number of newlines in leading content- stmtBaseLine = startToken.GetLine() - 1 - strings.Count(leadingContent, "\n")+ stmtBaseLine = base.GetLineOffset(startPosition) + startToken.GetLine() - 1 - strings.Count(leadingContent, "\n")prevStopTokenIndex = consumeTrailingSemicolon(tokens.GetAllTokens(), stopToken.GetTokenIndex())@@ -144,7 +179,7 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {}// Re-parse the individual statement with correct base line- stmtTree, stmtTokens, err := parsePLSQLInternal(stmtText, stmtBaseLine)+ stmtTree, stmtTokens, err := parsePLSQLInternal(stmtText, &storepb.Position{Line: int32(stmtBaseLine) + 1})if err != nil {return nil, err}@@ -162,13 +197,12 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {}// parsePLSQLInternal is the internal parsing function that parses a single SQL statement.-func parsePLSQLInternal(sql string, baseLine int) (antlr.Tree, *antlr.CommonTokenStream, error) {+func parsePLSQLInternal(sql string, startPosition *storepb.Position) (antlr.Tree, *antlr.CommonTokenStream, error) {lexer := parser.NewPlSqlLexer(antlr.NewInputStream(sql))stream := antlr.NewCommonTokenStream(lexer, 0)p := parser.NewPlSqlParser(stream)p.SetVersion12(true)- startPosition := &storepb.Position{Line: int32(baseLine) + 1}lexerErrorListener := &base.ParseErrorListener{Statement: sql,StartPosition: startPosition,@@ -201,7 +235,7 @@ func parsePLSQLInternal(sql string, baseLine int) (antlr.Tree, *antlr.CommonToke// This is used for strings manipulation which needs to see all statements together.func ParsePLSQLForStringsManipulation(sql string) (antlr.Tree, antlr.TokenStream, error) {sql = addSemicolonIfNeeded(sql)- return parsePLSQLInternal(sql, 0)+ return parsePLSQLInternal(sql, &storepb.Position{Line: 1})}func addSemicolonIfNeeded(sql string) string {diff --git a/backend/plugin/parser/plsql/plsql_test.go b/backend/plugin/parser/plsql/plsql_test.goindex c0f3505a7c..84720a0789 100644--- a/backend/plugin/parser/plsql/plsql_test.go+++ b/backend/plugin/parser/plsql/plsql_test.go@@ -1,11 +1,14 @@package plsqlimport (+ "errors""testing"+ "github.com/bytebase/omni/oracle/ast"parser "github.com/bytebase/parser/plsql""github.com/stretchr/testify/require"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -112,3 +115,78 @@ INSERT INTO t3 VALUES (1, 2);`,}}}++func TestParsePLSQLStatementsOmniFirst(t *testing.T) {+ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, "SELECT * FROM T;")+ require.NoError(t, err)+ require.Len(t, stmts, 1)++ omniAST, ok := stmts[0].AST.(*OmniAST)+ require.True(t, ok)+ require.Equal(t, "SELECT * FROM T", omniAST.Text)+ require.IsType(t, &ast.SelectStmt{}, omniAST.Node)+}++func TestParsePLSQLStatementsFallsBackToANTLR(t *testing.T) {+ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, `SELECT * FROM T;+CREATE TABLE GCP.LEAD_DROP_MC_NATIVE_DATA+(+ TXN_DATE DATE,+ USERID VARCHAR2(100),+ CUSTID VARCHAR2(100),+ SCREENID VARCHAR2(500),+ EVENTTIME DATE,+ STATUS NUMBER+)+PARTITION BY RANGE (TXN_DATE)+INTERVAL (NUMTODSINTERVAL(1,'DAY'))+(+ PARTITION P0 VALUES LESS THAN (DATE '2026-01-01')+);`)+ require.NoError(t, err)+ require.Len(t, stmts, 2)++ _, ok := stmts[0].AST.(*OmniAST)+ require.True(t, ok)++ _, ok = stmts[1].AST.(*base.ANTLRAST)+ require.True(t, ok)+}++func TestParsePLSQLStatementsFallbackErrorUsesOriginalLine(t *testing.T) {+ _, err := base.ParseStatements(storepb.Engine_ORACLE, `SELECT *+FROM T;+CREATE TABLE GCP.LEAD_DROP_MC_NATIVE_DATA+(+ TXN_DATE DATE+)+PARTITION BY RANGE (TXN_DATE)+INTERVAL (NUMTODSINTERVAL(1,'DAY'))+(+ PARTITION P0 VALUES LESS THAN (DATE '2026-01-01')+BROKEN`)+ require.Error(t, err)++ var syntaxErr *base.SyntaxError+ require.True(t, errors.As(err, &syntaxErr))+ require.NotNil(t, syntaxErr.Position)+ require.Equal(t, int32(11), syntaxErr.Position.Line)+}++func TestParsePLSQLStatementsFallbackErrorUsesOriginalColumn(t *testing.T) {+ statement := `SELECT * FROM T; CREATE TABLE GCP.LEAD_DROP_MC_NATIVE_DATA (TXN_DATE DATE) PARTITION BY RANGE (TXN_DATE) INTERVAL (NUMTODSINTERVAL(1,'DAY')) (PARTITION P0 VALUES LESS THAN (DATE '2026-01-01') BROKEN`++ _, wholeErr := ParsePLSQL(statement)+ require.Error(t, wholeErr)+ var wholeSyntaxErr *base.SyntaxError+ require.True(t, errors.As(wholeErr, &wholeSyntaxErr))+ require.NotNil(t, wholeSyntaxErr.Position)++ _, err := base.ParseStatements(storepb.Engine_ORACLE, statement)+ require.Error(t, err)+ var syntaxErr *base.SyntaxError+ require.True(t, errors.As(err, &syntaxErr))+ require.NotNil(t, syntaxErr.Position)+ require.Equal(t, wholeSyntaxErr.Position.Line, syntaxErr.Position.Line)+ require.Equal(t, wholeSyntaxErr.Position.Column, syntaxErr.Position.Column)+}