Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(pg): integrate omni parser for SQL splitting and AST interface (#19613)

rebelice7116a1c51fa1
diff --git a/backend/plugin/parser/pg/omni.go b/backend/plugin/parser/pg/omni.go
new file mode 100644
index 0000000000..7aece918a2
--- /dev/null
+++ b/backend/plugin/parser/pg/omni.go
@@ -0,0 +1,70 @@
+package pg
+
+import (
+ "unicode/utf8"
+
+ omnipg "github.com/bytebase/omni/pg"
+ "github.com/bytebase/omni/pg/ast"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// OmniAST wraps an omni AST node and implements the base.AST interface.
+type OmniAST struct {
+ // Node is the omni AST node (e.g. *ast.SelectStmt, *ast.CreateStmt).
+ Node ast.Node
+ // StartPosition is the 1-based position where this statement starts.
+ StartPosition *storepb.Position
+}
+
+// ASTStartPosition implements base.AST.
+func (a *OmniAST) ASTStartPosition() *storepb.Position {
+ return a.StartPosition
+}
+
+// ParsePg parses SQL using omni's parser and returns omni Statement objects directly.
+// This is the recommended entry point for new code that needs omni AST nodes.
+func ParsePg(sql string) ([]omnipg.Statement, error) {
+ return omnipg.Parse(sql)
+}
+
+// GetOmniNode extracts the omni AST node from a base.AST interface.
+// Returns the node and true if it is an OmniAST, nil and false otherwise.
+func GetOmniNode(a base.AST) (ast.Node, bool) {
+ if a == nil {
+ return nil, false
+ }
+ omniAST, ok := a.(*OmniAST)
+ if !ok {
+ return nil, false
+ }
+ return omniAST.Node, true
+}
+
+// byteOffsetToRunePosition converts a byte offset in sql to a 1-based line:column
+// where column is measured in Unicode code points (runes), matching storepb.Position semantics.
+func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {
+ if byteOffset > len(sql) {
+ byteOffset = len(sql)
+ }
+
+ line := int32(1)
+ runeCol := int32(0) // 0-based rune count on current line
+ i := 0
+ for i < byteOffset {
+ r, size := utf8.DecodeRuneInString(sql[i:])
+ if r == '\n' {
+ line++
+ runeCol = 0
+ } else {
+ runeCol++
+ }
+ i += size
+ }
+
+ return &storepb.Position{
+ Line: line,
+ Column: runeCol + 1, // convert to 1-based
+ }
+}
diff --git a/backend/plugin/parser/pg/pg.go b/backend/plugin/parser/pg/pg.go
index fbf6ba1129..d87573ccd8 100644
--- a/backend/plugin/parser/pg/pg.go
+++ b/backend/plugin/parser/pg/pg.go
@@ -17,33 +17,28 @@ func init() {
// parsePgStatements is the ParseStatementsFunc for PostgreSQL.
// Returns []ParsedStatement with both text and AST populated.
+// Uses omni's lexical splitter for splitting, then ANTLR for parsing each statement.
+// This preserves ANTLRAST compatibility for existing callers.
func parsePgStatements(statement string) ([]base.ParsedStatement, error) {
- // Split once to get Statement with text and positions
stmts, err := SplitSQL(statement)
if err != nil {
return nil, err
}
- // Parse using the pre-split statements to avoid double-splitting
- parseResults, err := parsePostgreSQLStatements(stmts)
- if err != nil {
- return nil, err
- }
-
- // Combine: Statement provides text/positions, ANTLRAST provides AST
var result []base.ParsedStatement
- astIndex := 0
for _, stmt := range stmts {
- ps := base.ParsedStatement{
- Statement: stmt,
+ if stmt.Empty {
+ continue
}
- if !stmt.Empty && astIndex < len(parseResults) {
- ps.AST = parseResults[astIndex]
- astIndex++
+ ast, err := parseSinglePostgreSQL(stmt.Text, stmt.BaseLine())
+ if err != nil {
+ return nil, err
}
- result = append(result, ps)
+ result = append(result, base.ParsedStatement{
+ Statement: stmt,
+ AST: ast,
+ })
}
-
return result, nil
}
diff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.go
index effdbc1e3d..9c3b3d2b67 100644
--- a/backend/plugin/parser/pg/split.go
+++ b/backend/plugin/parser/pg/split.go
@@ -1,14 +1,8 @@
package pg
import (
- "log/slog"
+ omnipg "github.com/bytebase/omni/pg"
- "github.com/antlr4-go/antlr/v4"
- "github.com/pkg/errors"
-
- parser "github.com/bytebase/parser/postgresql"
-
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -19,298 +13,23 @@ func init() {
}
// SplitSQL splits the given SQL statement into multiple SQL statements.
+// It uses omni's lexical splitter which handles quotes, dollar-quotes,
+// comments, and BEGIN ATOMIC blocks without requiring valid SQL.
func SplitSQL(statement string) ([]base.Statement, error) {
- lexer := parser.NewPostgreSQLLexer(antlr.NewInputStream(statement))
- stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
- list, err := splitSQLImpl(stream, statement)
- if err != nil {
- slog.Info("failed to split PostgreSQL statement", "statement", statement)
- // Use parser to split statement.
- return splitByParser(statement, lexer, stream)
- }
- return list, nil
-}
-
-func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlr.CommonTokenStream) ([]base.Statement, error) {
- p := parser.NewPostgreSQLParser(stream)
- lexerErrorListener := &base.ParseErrorListener{
- Statement: statement,
- }
- lexer.RemoveErrorListeners()
- lexer.AddErrorListener(lexerErrorListener)
-
- parserErrorListener := &base.ParseErrorListener{
- Statement: statement,
- }
- p.RemoveErrorListeners()
- p.AddErrorListener(parserErrorListener)
-
- p.BuildParseTrees = true
-
- tree := p.Root()
- if lexerErrorListener.Err != nil {
- return nil, lexerErrorListener.Err
- }
-
- if parserErrorListener.Err != nil {
- return nil, parserErrorListener.Err
- }
-
- if tree == nil || tree.Stmtblock() == nil || tree.Stmtblock().Stmtmulti() == nil {
- return nil, errors.New("failed to split multiple statements")
- }
-
- var result []base.Statement
- tokens := stream.GetAllTokens()
-
- byteOffset := 0
- start := 0
- for _, semi := range tree.Stmtblock().Stmtmulti().AllSEMI() {
- pos := semi.GetSymbol().GetStart()
- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
- stmtByteLength := len(stmtText)
+ segments := omnipg.Split(statement)
- // Calculate start position from byte offset (first character of Text)
- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
+ result := make([]base.Statement, 0, len(segments))
+ for _, seg := range segments {
result = append(result, base.Statement{
- Text: stmtText,
+ Text: seg.Text,
+ Empty: seg.Empty(),
+ Start: byteOffsetToRunePosition(statement, seg.ByteStart),
+ End: byteOffsetToRunePosition(statement, seg.ByteEnd),
Range: &storepb.Range{
- Start: int32(byteOffset),
- End: int32(byteOffset + stmtByteLength),
+ Start: int32(seg.ByteStart),
+ End: int32(seg.ByteEnd),
},
- End: common.ConvertANTLRTokenToExclusiveEndPosition(
- int32(tokens[pos].GetLine()),
- int32(tokens[pos].GetColumn()),
- tokens[pos].GetText(),
- ),
- Start: &storepb.Position{
- Line: int32(startLine + 1),
- Column: int32(startColumn + 1),
- },
- Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),
- })
- byteOffset += stmtByteLength
- start = pos + 1
- }
- // For the last statement, it may not end with semicolon symbol, EOF symbol instead.
- eofPos := len(tokens) - 1
- if start < eofPos {
- stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
- stmtByteLength := len(stmtText)
-
- // Calculate start position from byte offset (first character of Text)
- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
- result = append(result, base.Statement{
- Text: stmtText,
- Range: &storepb.Range{
- Start: int32(byteOffset),
- End: int32(byteOffset + stmtByteLength),
- },
- End: common.ConvertANTLRTokenToExclusiveEndPosition(
- int32(tokens[eofPos-1].GetLine()),
- int32(tokens[eofPos-1].GetColumn()),
- tokens[eofPos-1].GetText(),
- ),
- Start: &storepb.Position{
- Line: int32(startLine + 1),
- Column: int32(startColumn + 1),
- },
- Empty: base.IsEmpty(tokens[start:eofPos], parser.PostgreSQLParserSEMI),
})
}
return result, nil
}
-
-type openParenthesis struct {
- tokenType int
- pos int
-}
-
-func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Statement, error) {
- var result []base.Statement
- stream.Fill()
- tokens := stream.GetAllTokens()
-
- var beginCaseStack, ifStack, loopStack []openParenthesis
- var semicolonStack []int
-
- for i, token := range tokens {
- switch token.GetTokenType() {
- case parser.PostgreSQLParserBEGIN_P:
- if isBeginTransaction(tokens, i) {
- continue
- }
-
- beginCaseStack = append(beginCaseStack, openParenthesis{
- tokenType: token.GetTokenType(),
- pos: i,
- })
- case parser.PostgreSQLParserCASE:
- beginCaseStack = append(beginCaseStack, openParenthesis{
- tokenType: token.GetTokenType(),
- pos: i,
- })
- case parser.PostgreSQLParserIF_P:
- ifStack = append(ifStack, openParenthesis{
- tokenType: token.GetTokenType(),
- pos: i,
- })
- case parser.PostgreSQLParserLOOP:
- loopStack = append(loopStack, openParenthesis{
- tokenType: token.GetTokenType(),
- pos: i,
- })
- case parser.PostgreSQLParserSEMI:
- semicolonStack = append(semicolonStack, i)
- case parser.PostgreSQLParserEND_P:
- if isEndTransaction(tokens, i) {
- continue
- }
-
- nextToken := base.GetDefaultChannelTokenType(tokens, i, 1)
- switch nextToken {
- case parser.PostgreSQLParserIF_P:
- if len(ifStack) == 0 {
- return nil, errors.New("invalid statement: failed to split multiple statements")
- }
- // There are two cases:
- // 1. The IF statement with END IF statement.
- // 2. The IF statement without END IF statement.
- // We should match the longest IF statement, so we should check the first IF statement in the stack.
- semicolonStack = popSemicolonStack(semicolonStack, ifStack[0].pos)
- ifStack = ifStack[:len(ifStack)-1]
- case parser.PostgreSQLParserLOOP:
- if len(loopStack) == 0 {
- return nil, errors.New("invalid statement: failed to split multiple statements")
- }
- semicolonStack = popSemicolonStack(semicolonStack, loopStack[len(semicolonStack)-1].pos)
- loopStack = loopStack[:len(loopStack)-1]
- case parser.PostgreSQLParserCASE:
- if len(beginCaseStack) == 0 {
- return nil, errors.New("invalid statement: failed to split multiple statements")
- }
- semicolonStack = popSemicolonStack(semicolonStack, beginCaseStack[len(beginCaseStack)-1].pos)
- beginCaseStack = beginCaseStack[:len(beginCaseStack)-1]
- default:
- if len(beginCaseStack) == 0 {
- return nil, errors.New("invalid statement: failed to split multiple statements")
- }
- semicolonStack = popSemicolonStack(semicolonStack, beginCaseStack[len(beginCaseStack)-1].pos)
- beginCaseStack = beginCaseStack[:len(beginCaseStack)-1]
- }
- default:
- // Other tokens
- }
- }
-
- byteOffset := 0
- start := 0
- for _, pos := range semicolonStack {
- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
- stmtByteLength := len(stmtText)
-
- // Calculate start position from byte offset (first character of Text)
- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
- result = append(result, base.Statement{
- Text: stmtText,
- Range: &storepb.Range{
- Start: int32(byteOffset),
- End: int32(byteOffset + stmtByteLength),
- },
- End: common.ConvertANTLRTokenToExclusiveEndPosition(
- int32(tokens[pos].GetLine()),
- int32(tokens[pos].GetColumn()),
- tokens[pos].GetText(),
- ),
- Start: &storepb.Position{
- Line: int32(startLine + 1),
- Column: int32(startColumn + 1),
- },
- Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),
- })
- byteOffset += stmtByteLength
- start = pos + 1
- }
- // For the last statement, it may not end with semicolon symbol, EOF symbol instead.
- eofPos := len(tokens) - 1
- if start < eofPos {
- stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
- stmtByteLength := len(stmtText)
-
- // Calculate start position from byte offset (first character of Text)
- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
- result = append(result, base.Statement{
- Text: stmtText,
- Range: &storepb.Range{
- Start: int32(byteOffset),
- End: int32(byteOffset + stmtByteLength),
- },
- End: common.ConvertANTLRTokenToExclusiveEndPosition(
- int32(tokens[eofPos-1].GetLine()),
- int32(tokens[eofPos-1].GetColumn()),
- tokens[eofPos-1].GetText(),
- ),
- Start: &storepb.Position{
- Line: int32(startLine + 1),
- Column: int32(startColumn + 1),
- },
- Empty: base.IsEmpty(tokens[start:eofPos], parser.PostgreSQLParserSEMI),
- })
- }
-
- return result, nil
-}
-
-func popSemicolonStack(semicolonStack []int, pos int) []int {
- if len(semicolonStack) == 0 {
- return semicolonStack
- }
-
- for i := len(semicolonStack) - 1; i >= 0; i-- {
- if semicolonStack[i] < pos {
- return semicolonStack[:i+1]
- }
- }
-
- return []int{}
-}
-
-func isEndTransaction(tokens []antlr.Token, index int) bool {
- if tokens[index].GetTokenType() != parser.PostgreSQLLexerEND_P {
- return false
- }
-
- switch base.GetDefaultChannelTokenType(tokens, index, 1) {
- case parser.PostgreSQLParserTRANSACTION,
- parser.PostgreSQLParserWORK,
- parser.PostgreSQLParserSEMI:
- return true
- case parser.PostgreSQLParserAND:
- if base.GetDefaultChannelTokenType(tokens, index, 2) == parser.PostgreSQLParserNO {
- return base.GetDefaultChannelTokenType(tokens, index, 3) == parser.PostgreSQLParserCHAIN
- }
- return base.GetDefaultChannelTokenType(tokens, index, 2) == parser.PostgreSQLParserCHAIN
- default:
- return false
- }
-}
-
-func isBeginTransaction(tokens []antlr.Token, index int) bool {
- if tokens[index].GetTokenType() != parser.PostgreSQLLexerBEGIN_P {
- return false
- }
-
- switch base.GetDefaultChannelTokenType(tokens, index, 1) {
- case parser.PostgreSQLParserTRANSACTION,
- parser.PostgreSQLParserWORK,
- parser.PostgreSQLParserSEMI,
- parser.PostgreSQLParserISOLATION,
- parser.PostgreSQLParserREAD,
- parser.PostgreSQLLexerDEFERRABLE:
- return true
- case parser.PostgreSQLParserNOT:
- return base.GetDefaultChannelTokenType(tokens, index, 2) == parser.PostgreSQLParserDEFERRABLE
- default:
- return false
- }
-}
diff --git a/go.mod b/go.mod
index 56e3e8fd1d..f009a59647 100644
--- a/go.mod
+++ b/go.mod
@@ -34,6 +34,7 @@ require (
github.com/bmatcuk/doublestar/v4 v4.10.0
github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0
+ github.com/bytebase/omni v0.0.0-20260313074616-3f695e9ee28c
github.com/caarlos0/env/v11 v11.4.0
github.com/cenkalti/backoff/v5 v5.0.3
github.com/cockroachdb/cockroachdb-parser v0.25.2
diff --git a/go.sum b/go.sum
index 41f69b7870..435c1db61e 100644
--- a/go.sum
+++ b/go.sum
@@ -253,6 +253,8 @@ github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65 h1:RvViQx0e24DeDJ
github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65/go.mod h1:4dDI0RO2BK9/BD4VE+YqvlCj/qoVMquZ99QZdRlT8Wg=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=
+github.com/bytebase/omni v0.0.0-20260313074616-3f695e9ee28c h1:+snIwvonEDmEhP0ypzkJhJKM/ocDqjJlVHrhcTTqG0I=
+github.com/bytebase/omni v0.0.0-20260313074616-3f695e9ee28c/go.mod h1:sHnw/Do16VheYa+ThYDBzY98EbkwFwWBzxh2CQuOgD0=
github.com/bytebase/parser v0.0.0-20260228025849-39c76b45c98a h1:hTyZhniv0HzQYM2K6mG1XpOEKibsfLJ7jYhBf/1sXLc=
github.com/bytebase/parser v0.0.0-20260228025849-39c76b45c98a/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=
github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=