Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(pg): integrate omni parser for SQL splitting and AST interface (#19613)
rebelice7116a1c51fa1
diff --git a/backend/plugin/parser/pg/omni.go b/backend/plugin/parser/pg/omni.gonew file mode 100644index 0000000000..7aece918a2--- /dev/null+++ b/backend/plugin/parser/pg/omni.go@@ -0,0 +1,70 @@+package pg++import (+ "unicode/utf8"++ omnipg "github.com/bytebase/omni/pg"+ "github.com/bytebase/omni/pg/ast"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// OmniAST wraps an omni AST node and implements the base.AST interface.+type OmniAST struct {+ // Node is the omni AST node (e.g. *ast.SelectStmt, *ast.CreateStmt).+ Node ast.Node+ // StartPosition is the 1-based position where this statement starts.+ StartPosition *storepb.Position+}++// ASTStartPosition implements base.AST.+func (a *OmniAST) ASTStartPosition() *storepb.Position {+ return a.StartPosition+}++// ParsePg parses SQL using omni's parser and returns omni Statement objects directly.+// This is the recommended entry point for new code that needs omni AST nodes.+func ParsePg(sql string) ([]omnipg.Statement, error) {+ return omnipg.Parse(sql)+}++// GetOmniNode extracts the omni AST node from a base.AST interface.+// Returns the node and true if it is an OmniAST, nil and false otherwise.+func GetOmniNode(a base.AST) (ast.Node, bool) {+ if a == nil {+ return nil, false+ }+ omniAST, ok := a.(*OmniAST)+ if !ok {+ return nil, false+ }+ return omniAST.Node, true+}++// byteOffsetToRunePosition converts a byte offset in sql to a 1-based line:column+// where column is measured in Unicode code points (runes), matching storepb.Position semantics.+func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {+ if byteOffset > len(sql) {+ byteOffset = len(sql)+ }++ line := int32(1)+ runeCol := int32(0) // 0-based rune count on current line+ i := 0+ for i < byteOffset {+ r, size := utf8.DecodeRuneInString(sql[i:])+ if r == '\n' {+ line+++ runeCol = 0+ } else {+ runeCol+++ }+ i += size+ }++ return &storepb.Position{+ Line: line,+ Column: runeCol + 1, // convert to 1-based+ }+}diff --git a/backend/plugin/parser/pg/pg.go b/backend/plugin/parser/pg/pg.goindex fbf6ba1129..d87573ccd8 100644--- a/backend/plugin/parser/pg/pg.go+++ b/backend/plugin/parser/pg/pg.go@@ -17,33 +17,28 @@ func init() {// parsePgStatements is the ParseStatementsFunc for PostgreSQL.// Returns []ParsedStatement with both text and AST populated.+// Uses omni's lexical splitter for splitting, then ANTLR for parsing each statement.+// This preserves ANTLRAST compatibility for existing callers.func parsePgStatements(statement string) ([]base.ParsedStatement, error) {- // Split once to get Statement with text and positionsstmts, err := SplitSQL(statement)if err != nil {return nil, err}- // Parse using the pre-split statements to avoid double-splitting- parseResults, err := parsePostgreSQLStatements(stmts)- if err != nil {- return nil, err- }-- // Combine: Statement provides text/positions, ANTLRAST provides ASTvar result []base.ParsedStatement- astIndex := 0for _, stmt := range stmts {- ps := base.ParsedStatement{- Statement: stmt,+ if stmt.Empty {+ continue}- if !stmt.Empty && astIndex < len(parseResults) {- ps.AST = parseResults[astIndex]- astIndex+++ ast, err := parseSinglePostgreSQL(stmt.Text, stmt.BaseLine())+ if err != nil {+ return nil, err}- result = append(result, ps)+ result = append(result, base.ParsedStatement{+ Statement: stmt,+ AST: ast,+ })}-return result, nil}diff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.goindex effdbc1e3d..9c3b3d2b67 100644--- a/backend/plugin/parser/pg/split.go+++ b/backend/plugin/parser/pg/split.go@@ -1,14 +1,8 @@package pgimport (- "log/slog"+ omnipg "github.com/bytebase/omni/pg"- "github.com/antlr4-go/antlr/v4"- "github.com/pkg/errors"-- parser "github.com/bytebase/parser/postgresql"-- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -19,298 +13,23 @@ func init() {}// SplitSQL splits the given SQL statement into multiple SQL statements.+// It uses omni's lexical splitter which handles quotes, dollar-quotes,+// comments, and BEGIN ATOMIC blocks without requiring valid SQL.func SplitSQL(statement string) ([]base.Statement, error) {- lexer := parser.NewPostgreSQLLexer(antlr.NewInputStream(statement))- stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)- list, err := splitSQLImpl(stream, statement)- if err != nil {- slog.Info("failed to split PostgreSQL statement", "statement", statement)- // Use parser to split statement.- return splitByParser(statement, lexer, stream)- }- return list, nil-}--func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlr.CommonTokenStream) ([]base.Statement, error) {- p := parser.NewPostgreSQLParser(stream)- lexerErrorListener := &base.ParseErrorListener{- Statement: statement,- }- lexer.RemoveErrorListeners()- lexer.AddErrorListener(lexerErrorListener)-- parserErrorListener := &base.ParseErrorListener{- Statement: statement,- }- p.RemoveErrorListeners()- p.AddErrorListener(parserErrorListener)-- p.BuildParseTrees = true-- tree := p.Root()- if lexerErrorListener.Err != nil {- return nil, lexerErrorListener.Err- }-- if parserErrorListener.Err != nil {- return nil, parserErrorListener.Err- }-- if tree == nil || tree.Stmtblock() == nil || tree.Stmtblock().Stmtmulti() == nil {- return nil, errors.New("failed to split multiple statements")- }-- var result []base.Statement- tokens := stream.GetAllTokens()-- byteOffset := 0- start := 0- for _, semi := range tree.Stmtblock().Stmtmulti().AllSEMI() {- pos := semi.GetSymbol().GetStart()- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)+ segments := omnipg.Split(statement)- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)+ result := make([]base.Statement, 0, len(segments))+ for _, seg := range segments {result = append(result, base.Statement{- Text: stmtText,+ Text: seg.Text,+ Empty: seg.Empty(),+ Start: byteOffsetToRunePosition(statement, seg.ByteStart),+ End: byteOffsetToRunePosition(statement, seg.ByteEnd),Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),+ Start: int32(seg.ByteStart),+ End: int32(seg.ByteEnd),},- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),- })- byteOffset += stmtByteLength- start = pos + 1- }- // For the last statement, it may not end with semicolon symbol, EOF symbol instead.- eofPos := len(tokens) - 1- if start < eofPos {- stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])- stmtByteLength := len(stmtText)-- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[eofPos-1].GetLine()),- int32(tokens[eofPos-1].GetColumn()),- tokens[eofPos-1].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: base.IsEmpty(tokens[start:eofPos], parser.PostgreSQLParserSEMI),})}return result, nil}--type openParenthesis struct {- tokenType int- pos int-}--func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Statement, error) {- var result []base.Statement- stream.Fill()- tokens := stream.GetAllTokens()-- var beginCaseStack, ifStack, loopStack []openParenthesis- var semicolonStack []int-- for i, token := range tokens {- switch token.GetTokenType() {- case parser.PostgreSQLParserBEGIN_P:- if isBeginTransaction(tokens, i) {- continue- }-- beginCaseStack = append(beginCaseStack, openParenthesis{- tokenType: token.GetTokenType(),- pos: i,- })- case parser.PostgreSQLParserCASE:- beginCaseStack = append(beginCaseStack, openParenthesis{- tokenType: token.GetTokenType(),- pos: i,- })- case parser.PostgreSQLParserIF_P:- ifStack = append(ifStack, openParenthesis{- tokenType: token.GetTokenType(),- pos: i,- })- case parser.PostgreSQLParserLOOP:- loopStack = append(loopStack, openParenthesis{- tokenType: token.GetTokenType(),- pos: i,- })- case parser.PostgreSQLParserSEMI:- semicolonStack = append(semicolonStack, i)- case parser.PostgreSQLParserEND_P:- if isEndTransaction(tokens, i) {- continue- }-- nextToken := base.GetDefaultChannelTokenType(tokens, i, 1)- switch nextToken {- case parser.PostgreSQLParserIF_P:- if len(ifStack) == 0 {- return nil, errors.New("invalid statement: failed to split multiple statements")- }- // There are two cases:- // 1. The IF statement with END IF statement.- // 2. The IF statement without END IF statement.- // We should match the longest IF statement, so we should check the first IF statement in the stack.- semicolonStack = popSemicolonStack(semicolonStack, ifStack[0].pos)- ifStack = ifStack[:len(ifStack)-1]- case parser.PostgreSQLParserLOOP:- if len(loopStack) == 0 {- return nil, errors.New("invalid statement: failed to split multiple statements")- }- semicolonStack = popSemicolonStack(semicolonStack, loopStack[len(semicolonStack)-1].pos)- loopStack = loopStack[:len(loopStack)-1]- case parser.PostgreSQLParserCASE:- if len(beginCaseStack) == 0 {- return nil, errors.New("invalid statement: failed to split multiple statements")- }- semicolonStack = popSemicolonStack(semicolonStack, beginCaseStack[len(beginCaseStack)-1].pos)- beginCaseStack = beginCaseStack[:len(beginCaseStack)-1]- default:- if len(beginCaseStack) == 0 {- return nil, errors.New("invalid statement: failed to split multiple statements")- }- semicolonStack = popSemicolonStack(semicolonStack, beginCaseStack[len(beginCaseStack)-1].pos)- beginCaseStack = beginCaseStack[:len(beginCaseStack)-1]- }- default:- // Other tokens- }- }-- byteOffset := 0- start := 0- for _, pos := range semicolonStack {- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)-- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),- })- byteOffset += stmtByteLength- start = pos + 1- }- // For the last statement, it may not end with semicolon symbol, EOF symbol instead.- eofPos := len(tokens) - 1- if start < eofPos {- stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])- stmtByteLength := len(stmtText)-- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[eofPos-1].GetLine()),- int32(tokens[eofPos-1].GetColumn()),- tokens[eofPos-1].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: base.IsEmpty(tokens[start:eofPos], parser.PostgreSQLParserSEMI),- })- }-- return result, nil-}--func popSemicolonStack(semicolonStack []int, pos int) []int {- if len(semicolonStack) == 0 {- return semicolonStack- }-- for i := len(semicolonStack) - 1; i >= 0; i-- {- if semicolonStack[i] < pos {- return semicolonStack[:i+1]- }- }-- return []int{}-}--func isEndTransaction(tokens []antlr.Token, index int) bool {- if tokens[index].GetTokenType() != parser.PostgreSQLLexerEND_P {- return false- }-- switch base.GetDefaultChannelTokenType(tokens, index, 1) {- case parser.PostgreSQLParserTRANSACTION,- parser.PostgreSQLParserWORK,- parser.PostgreSQLParserSEMI:- return true- case parser.PostgreSQLParserAND:- if base.GetDefaultChannelTokenType(tokens, index, 2) == parser.PostgreSQLParserNO {- return base.GetDefaultChannelTokenType(tokens, index, 3) == parser.PostgreSQLParserCHAIN- }- return base.GetDefaultChannelTokenType(tokens, index, 2) == parser.PostgreSQLParserCHAIN- default:- return false- }-}--func isBeginTransaction(tokens []antlr.Token, index int) bool {- if tokens[index].GetTokenType() != parser.PostgreSQLLexerBEGIN_P {- return false- }-- switch base.GetDefaultChannelTokenType(tokens, index, 1) {- case parser.PostgreSQLParserTRANSACTION,- parser.PostgreSQLParserWORK,- parser.PostgreSQLParserSEMI,- parser.PostgreSQLParserISOLATION,- parser.PostgreSQLParserREAD,- parser.PostgreSQLLexerDEFERRABLE:- return true- case parser.PostgreSQLParserNOT:- return base.GetDefaultChannelTokenType(tokens, index, 2) == parser.PostgreSQLParserDEFERRABLE- default:- return false- }-}diff --git a/go.mod b/go.modindex 56e3e8fd1d..f009a59647 100644--- a/go.mod+++ b/go.mod@@ -34,6 +34,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0+ github.com/bytebase/omni v0.0.0-20260313074616-3f695e9ee28cgithub.com/caarlos0/env/v11 v11.4.0github.com/cenkalti/backoff/v5 v5.0.3github.com/cockroachdb/cockroachdb-parser v0.25.2diff --git a/go.sum b/go.sumindex 41f69b7870..435c1db61e 100644--- a/go.sum+++ b/go.sum@@ -253,6 +253,8 @@ github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65 h1:RvViQx0e24DeDJgithub.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65/go.mod h1:4dDI0RO2BK9/BD4VE+YqvlCj/qoVMquZ99QZdRlT8Wg=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=+github.com/bytebase/omni v0.0.0-20260313074616-3f695e9ee28c h1:+snIwvonEDmEhP0ypzkJhJKM/ocDqjJlVHrhcTTqG0I=+github.com/bytebase/omni v0.0.0-20260313074616-3f695e9ee28c/go.mod h1:sHnw/Do16VheYa+ThYDBzY98EbkwFwWBzxh2CQuOgD0=github.com/bytebase/parser v0.0.0-20260228025849-39c76b45c98a h1:hTyZhniv0HzQYM2K6mG1XpOEKibsfLJ7jYhBf/1sXLc=github.com/bytebase/parser v0.0.0-20260228025849-39c76b45c98a/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=