Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

chore(plsql): add omni-first parse dispatcher (#20349)

rebelice3556a1d7d167
diff --git a/backend/plugin/parser/base/base.go b/backend/plugin/parser/base/base.go
index 9383d03b84..b706316421 100644
--- a/backend/plugin/parser/base/base.go
+++ b/backend/plugin/parser/base/base.go
@@ -37,8 +37,12 @@ func (l *ParseErrorListener) SyntaxError(_ antlr.Recognizer, token any, line, co
// Get 0-based line offset from StartPosition (1-based) for calculations
lineOffset := int32(0)
+ columnOffset := int32(0)
if l.StartPosition != nil {
lineOffset = l.StartPosition.Line - 1
+ if line == 1 && l.StartPosition.Column > 0 {
+ columnOffset = l.StartPosition.Column - 1
+ }
}
errMessage := ""
@@ -57,7 +61,7 @@ func (l *ParseErrorListener) SyntaxError(_ antlr.Recognizer, token any, line, co
// ANTLR provides 1-based line and 0-based column
// Store as 1-based line and 1-based column in Position
posLine := int32(line) + lineOffset
- posColumn := int32(column + 1)
+ posColumn := int32(column+1) + columnOffset
l.Err = &SyntaxError{
Position: &storepb.Position{
Line: posLine,
diff --git a/backend/plugin/parser/plsql/omni.go b/backend/plugin/parser/plsql/omni.go
index 3968e3da6e..af7fe4eb2c 100644
--- a/backend/plugin/parser/plsql/omni.go
+++ b/backend/plugin/parser/plsql/omni.go
@@ -3,14 +3,18 @@ package plsql
import (
"unicode/utf8"
+ "github.com/antlr4-go/antlr/v4"
"github.com/bytebase/omni/oracle/ast"
oracleparser "github.com/bytebase/omni/oracle/parser"
+ antlrparser "github.com/bytebase/parser/plsql"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
// OmniAST wraps an omni AST node and implements the base.AST interface.
+// During migration, it also implements AntlrASTProvider so that callers
+// still using GetANTLRAST() can fall back to the ANTLR tree.
type OmniAST struct {
// Node is the omni AST node (e.g. *ast.SelectStmt, *ast.CreateTableStmt).
Node ast.Node
@@ -18,6 +22,12 @@ type OmniAST struct {
Text string
// StartPosition is the 1-based position where this statement starts.
StartPosition *storepb.Position
+
+ // antlrAST is lazily populated when AsANTLRAST() is called.
+ // This field will be removed once all Oracle modules are migrated to omni.
+ antlrAST *base.ANTLRAST
+ // antlrParsed tracks whether we've attempted the ANTLR parse.
+ antlrParsed bool
}
// ASTStartPosition implements base.AST.
@@ -25,6 +35,41 @@ func (a *OmniAST) ASTStartPosition() *storepb.Position {
return a.StartPosition
}
+// AsANTLRAST implements base.AntlrASTProvider for backward compatibility.
+// It lazily parses the SQL text with the ANTLR parser and caches the result.
+func (a *OmniAST) AsANTLRAST() (*base.ANTLRAST, bool) {
+ if a.antlrParsed {
+ return a.antlrAST, a.antlrAST != nil
+ }
+ a.antlrParsed = true
+
+ tree, tokens := parseSinglePLSQLLenient(a.Text)
+ a.antlrAST = &base.ANTLRAST{
+ StartPosition: a.StartPosition,
+ Tree: tree,
+ Tokens: tokens,
+ }
+ return a.antlrAST, true
+}
+
+// parseSinglePLSQLLenient parses a single Oracle statement without error listeners.
+// The omni parser has already validated the SQL; this tree only exists for
+// backward-compatible ANTLR consumers during the migration window.
+func parseSinglePLSQLLenient(statement string) (antlr.Tree, *antlr.CommonTokenStream) {
+ inputStream := antlr.NewInputStream(addSemicolonIfNeeded(statement))
+ lexer := antlrparser.NewPlSqlLexer(inputStream)
+ stream := antlr.NewCommonTokenStream(lexer, 0)
+ p := antlrparser.NewPlSqlParser(stream)
+ p.SetVersion12(true)
+
+ lexer.RemoveErrorListeners()
+ p.RemoveErrorListeners()
+ p.BuildParseTrees = true
+
+ tree := p.Sql_script()
+ return tree, stream
+}
+
// ParsePLSQLOmni parses SQL using omni's parser and returns an ast.List.
// This is the recommended entry point for new Oracle code that needs omni AST nodes.
func ParsePLSQLOmni(sql string) (*ast.List, error) {
diff --git a/backend/plugin/parser/plsql/omni_test.go b/backend/plugin/parser/plsql/omni_test.go
index abb917b488..cf4cf5875f 100644
--- a/backend/plugin/parser/plsql/omni_test.go
+++ b/backend/plugin/parser/plsql/omni_test.go
@@ -3,6 +3,7 @@ package plsql
import (
"testing"
+ parser "github.com/bytebase/parser/plsql"
"github.com/stretchr/testify/require"
"github.com/bytebase/omni/oracle/ast"
@@ -52,6 +53,25 @@ func TestOracleOmniASTWrapper(t *testing.T) {
require.Nil(t, got)
}
+func TestOracleOmniASTAsANTLRAST(t *testing.T) {
+ start := &storepb.Position{Line: 4, Column: 1}
+ omniAST := &OmniAST{
+ Node: &ast.SelectStmt{},
+ Text: "SELECT * FROM T",
+ StartPosition: start,
+ }
+
+ antlrAST, ok := base.GetANTLRAST(omniAST)
+ require.True(t, ok)
+ require.Equal(t, start, antlrAST.StartPosition)
+ require.IsType(t, &parser.Sql_scriptContext{}, antlrAST.Tree)
+ require.NotNil(t, antlrAST.Tokens)
+
+ antlrASTAgain, ok := base.GetANTLRAST(omniAST)
+ require.True(t, ok)
+ require.Same(t, antlrAST, antlrASTAgain)
+}
+
func TestOracleByteOffsetToRunePosition(t *testing.T) {
position := ByteOffsetToRunePosition("SELECT\n日本 FROM DUAL", len("SELECT\n日"))
require.Equal(t, int32(2), position.Line)
diff --git a/backend/plugin/parser/plsql/plsql.go b/backend/plugin/parser/plsql/plsql.go
index 0cc11ce1d2..f193401e30 100644
--- a/backend/plugin/parser/plsql/plsql.go
+++ b/backend/plugin/parser/plsql/plsql.go
@@ -7,6 +7,7 @@ import (
"strings"
"github.com/antlr4-go/antlr/v4"
+ "github.com/bytebase/omni/oracle/ast"
parser "github.com/bytebase/parser/plsql"
"github.com/pkg/errors"
@@ -28,24 +29,50 @@ func parsePLSQLStatements(statement string) ([]base.ParsedStatement, error) {
return nil, err
}
- // Then parse to get ASTs (note: ParsePLSQL adds semicolon internally)
- parseResults, err := ParsePLSQL(statement + ";")
- if err != nil {
- return nil, err
- }
-
- // Combine: Statement provides text/positions, ANTLRAST provides AST
var result []base.ParsedStatement
- astIndex := 0
for _, stmt := range stmts {
- ps := base.ParsedStatement{
- Statement: stmt,
+ if stmt.Empty {
+ result = append(result, base.ParsedStatement{Statement: stmt})
+ continue
+ }
+
+ list, omniErr := ParsePLSQLOmni(stmt.Text)
+ if omniErr == nil {
+ if list == nil || len(list.Items) == 0 {
+ result = append(result, base.ParsedStatement{Statement: stmt})
+ continue
+ }
+ for _, node := range list.Items {
+ raw, ok := node.(*ast.RawStmt)
+ if !ok {
+ continue
+ }
+ result = append(result, base.ParsedStatement{
+ Statement: stmt,
+ AST: &OmniAST{
+ Node: raw.Stmt,
+ Text: stmt.Text,
+ StartPosition: stmt.Start,
+ },
+ })
+ }
+ continue
+ }
+
+ parseResults, err := parsePLSQLWithStartPosition(stmt.Text, stmt.Start)
+ if err != nil {
+ return nil, err
+ }
+ if len(parseResults) == 0 {
+ result = append(result, base.ParsedStatement{Statement: stmt})
+ continue
}
- if !stmt.Empty && astIndex < len(parseResults) {
- ps.AST = parseResults[astIndex]
- astIndex++
+ for _, parseResult := range parseResults {
+ result = append(result, base.ParsedStatement{
+ Statement: stmt,
+ AST: parseResult,
+ })
}
- result = append(result, ps)
}
return result, nil
@@ -88,10 +115,18 @@ func ParseVersion(banner string) (*Version, error) {
// It first parses the whole statement to get the AST, then splits by unit_statement
// and sql_plus_command nodes, and re-parses each individual statement.
func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {
+ return parsePLSQLWithBaseLine(sql, 0)
+}
+
+func parsePLSQLWithBaseLine(sql string, baseLine int) ([]*base.ANTLRAST, error) {
+ return parsePLSQLWithStartPosition(sql, &storepb.Position{Line: int32(baseLine) + 1})
+}
+
+func parsePLSQLWithStartPosition(sql string, startPosition *storepb.Position) ([]*base.ANTLRAST, error) {
sql = addSemicolonIfNeeded(sql)
// First pass: parse the whole statement to get the AST for splitting
- tree, tokens, err := parsePLSQLInternal(sql, 0)
+ tree, tokens, err := parsePLSQLInternal(sql, startPosition)
if err != nil {
return nil, err
}
@@ -134,7 +169,7 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {
// stmtBaseLine is where the leading content starts (for re-parsing with correct offsets).
// This ensures token positions in the re-parsed AST are correct when combined with SplitSQL's BaseLine.
// Formula: first token's line - 1 (convert to 0-based) - number of newlines in leading content
- stmtBaseLine = startToken.GetLine() - 1 - strings.Count(leadingContent, "\n")
+ stmtBaseLine = base.GetLineOffset(startPosition) + startToken.GetLine() - 1 - strings.Count(leadingContent, "\n")
prevStopTokenIndex = consumeTrailingSemicolon(tokens.GetAllTokens(), stopToken.GetTokenIndex())
@@ -144,7 +179,7 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {
}
// Re-parse the individual statement with correct base line
- stmtTree, stmtTokens, err := parsePLSQLInternal(stmtText, stmtBaseLine)
+ stmtTree, stmtTokens, err := parsePLSQLInternal(stmtText, &storepb.Position{Line: int32(stmtBaseLine) + 1})
if err != nil {
return nil, err
}
@@ -162,13 +197,12 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {
}
// parsePLSQLInternal is the internal parsing function that parses a single SQL statement.
-func parsePLSQLInternal(sql string, baseLine int) (antlr.Tree, *antlr.CommonTokenStream, error) {
+func parsePLSQLInternal(sql string, startPosition *storepb.Position) (antlr.Tree, *antlr.CommonTokenStream, error) {
lexer := parser.NewPlSqlLexer(antlr.NewInputStream(sql))
stream := antlr.NewCommonTokenStream(lexer, 0)
p := parser.NewPlSqlParser(stream)
p.SetVersion12(true)
- startPosition := &storepb.Position{Line: int32(baseLine) + 1}
lexerErrorListener := &base.ParseErrorListener{
Statement: sql,
StartPosition: startPosition,
@@ -201,7 +235,7 @@ func parsePLSQLInternal(sql string, baseLine int) (antlr.Tree, *antlr.CommonToke
// This is used for strings manipulation which needs to see all statements together.
func ParsePLSQLForStringsManipulation(sql string) (antlr.Tree, antlr.TokenStream, error) {
sql = addSemicolonIfNeeded(sql)
- return parsePLSQLInternal(sql, 0)
+ return parsePLSQLInternal(sql, &storepb.Position{Line: 1})
}
func addSemicolonIfNeeded(sql string) string {
diff --git a/backend/plugin/parser/plsql/plsql_test.go b/backend/plugin/parser/plsql/plsql_test.go
index c0f3505a7c..84720a0789 100644
--- a/backend/plugin/parser/plsql/plsql_test.go
+++ b/backend/plugin/parser/plsql/plsql_test.go
@@ -1,11 +1,14 @@
package plsql
import (
+ "errors"
"testing"
+ "github.com/bytebase/omni/oracle/ast"
parser "github.com/bytebase/parser/plsql"
"github.com/stretchr/testify/require"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -112,3 +115,78 @@ INSERT INTO t3 VALUES (1, 2);`,
}
}
}
+
+func TestParsePLSQLStatementsOmniFirst(t *testing.T) {
+ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, "SELECT * FROM T;")
+ require.NoError(t, err)
+ require.Len(t, stmts, 1)
+
+ omniAST, ok := stmts[0].AST.(*OmniAST)
+ require.True(t, ok)
+ require.Equal(t, "SELECT * FROM T", omniAST.Text)
+ require.IsType(t, &ast.SelectStmt{}, omniAST.Node)
+}
+
+func TestParsePLSQLStatementsFallsBackToANTLR(t *testing.T) {
+ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, `SELECT * FROM T;
+CREATE TABLE GCP.LEAD_DROP_MC_NATIVE_DATA
+(
+ TXN_DATE DATE,
+ USERID VARCHAR2(100),
+ CUSTID VARCHAR2(100),
+ SCREENID VARCHAR2(500),
+ EVENTTIME DATE,
+ STATUS NUMBER
+)
+PARTITION BY RANGE (TXN_DATE)
+INTERVAL (NUMTODSINTERVAL(1,'DAY'))
+(
+ PARTITION P0 VALUES LESS THAN (DATE '2026-01-01')
+);`)
+ require.NoError(t, err)
+ require.Len(t, stmts, 2)
+
+ _, ok := stmts[0].AST.(*OmniAST)
+ require.True(t, ok)
+
+ _, ok = stmts[1].AST.(*base.ANTLRAST)
+ require.True(t, ok)
+}
+
+func TestParsePLSQLStatementsFallbackErrorUsesOriginalLine(t *testing.T) {
+ _, err := base.ParseStatements(storepb.Engine_ORACLE, `SELECT *
+FROM T;
+CREATE TABLE GCP.LEAD_DROP_MC_NATIVE_DATA
+(
+ TXN_DATE DATE
+)
+PARTITION BY RANGE (TXN_DATE)
+INTERVAL (NUMTODSINTERVAL(1,'DAY'))
+(
+ PARTITION P0 VALUES LESS THAN (DATE '2026-01-01')
+BROKEN`)
+ require.Error(t, err)
+
+ var syntaxErr *base.SyntaxError
+ require.True(t, errors.As(err, &syntaxErr))
+ require.NotNil(t, syntaxErr.Position)
+ require.Equal(t, int32(11), syntaxErr.Position.Line)
+}
+
+func TestParsePLSQLStatementsFallbackErrorUsesOriginalColumn(t *testing.T) {
+ statement := `SELECT * FROM T; CREATE TABLE GCP.LEAD_DROP_MC_NATIVE_DATA (TXN_DATE DATE) PARTITION BY RANGE (TXN_DATE) INTERVAL (NUMTODSINTERVAL(1,'DAY')) (PARTITION P0 VALUES LESS THAN (DATE '2026-01-01') BROKEN`
+
+ _, wholeErr := ParsePLSQL(statement)
+ require.Error(t, wholeErr)
+ var wholeSyntaxErr *base.SyntaxError
+ require.True(t, errors.As(wholeErr, &wholeSyntaxErr))
+ require.NotNil(t, wholeSyntaxErr.Position)
+
+ _, err := base.ParseStatements(storepb.Engine_ORACLE, statement)
+ require.Error(t, err)
+ var syntaxErr *base.SyntaxError
+ require.True(t, errors.As(err, &syntaxErr))
+ require.NotNil(t, syntaxErr.Position)
+ require.Equal(t, wholeSyntaxErr.Position.Line, syntaxErr.Position.Line)
+ require.Equal(t, wholeSyntaxErr.Position.Column, syntaxErr.Position.Column)
+}