Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: create grammar-free SplitSQLByLexer helper for ANTLR-based splitters (#18158)
h3n4l2bc565ada160
diff --git a/backend/plugin/parser/base/lexer.go b/backend/plugin/parser/base/lexer.goindex f602a45474..812d8e7536 100644--- a/backend/plugin/parser/base/lexer.go+++ b/backend/plugin/parser/base/lexer.go@@ -1,6 +1,8 @@package baseimport (+ "strings"+"github.com/antlr4-go/antlr/v4""github.com/bytebase/bytebase/backend/common"@@ -53,3 +55,69 @@ func IsEmpty(tokens []antlr.Token, semi int) bool {}return true}++// SplitSQLByLexer is a grammar-free helper function that splits SQL statements using an ANTLR lexer.+// It works with any ANTLR-based lexer by accepting the token stream and semicolon token type as parameters.+//+// Parameters:+// - stream: The ANTLR token stream (must be already filled with stream.Fill())+// - semiTokenType: The token type value for semicolon in the specific grammar+// - statement: The original SQL statement string (used for position conversion)+//+// Returns:+// - A slice of SingleSQL, each representing one statement with its text, position, and metadata+//+// Example usage:+//+// lexer := parser.NewSnowflakeLexer(antlr.NewInputStream(statement))+// stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)+// stream.Fill()+// return SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI, statement)+func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, statement string) ([]SingleSQL, error) {+ tokens := stream.GetAllTokens()+ var buf []antlr.Token+ var sqls []SingleSQL++ for i, token := range tokens {+ // Collect all tokens except the last one (EOF)+ if i < len(tokens)-1 {+ buf = append(buf, token)+ }++ // Split on semicolon or at the end of tokens+ if (token.GetTokenType() == semiTokenType || i == len(tokens)-1) && len(buf) > 0 {+ // Reconstruct the text from tokens+ bufStr := new(strings.Builder)+ empty := true++ for _, b := range buf {+ if _, err := bufStr.WriteString(b.GetText()); err != nil {+ return nil, err+ }+ // Check if there's any non-hidden token (actual code, not just comments/whitespace)+ if b.GetChannel() != antlr.TokenHiddenChannel {+ empty = false+ }+ }++ // Get the position of the first default channel token for Start position+ antlrPosition := FirstDefaultChannelTokenPosition(buf)++ sqls = append(sqls, SingleSQL{+ Text: bufStr.String(),+ BaseLine: buf[0].GetLine() - 1, // BaseLine is the offset of the first token+ End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{+ Line: int32(token.GetLine()),+ Column: int32(token.GetColumn()),+ }, statement),+ Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Empty: empty,+ })++ buf = nil+ continue+ }+ }++ return sqls, nil+}diff --git a/backend/plugin/parser/cassandra/split.go b/backend/plugin/parser/cassandra/split.goindex 2a44a40467..a3517e251f 100644--- a/backend/plugin/parser/cassandra/split.go+++ b/backend/plugin/parser/cassandra/split.go@@ -1,12 +1,9 @@package cassandraimport (- "strings"-"github.com/antlr4-go/antlr/v4""github.com/bytebase/parser/cql"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -21,43 +18,5 @@ func SplitSQL(statement string) ([]base.SingleSQL, error) {stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()- tokens := stream.GetAllTokens()- var buf []antlr.Token- var sqls []base.SingleSQL-- for i, token := range tokens {- if i < len(tokens)-1 {- buf = append(buf, token)- }- if (token.GetTokenType() == cql.CqlLexerSEMI || i == len(tokens)-1) && len(buf) > 0 {- bufStr := new(strings.Builder)- empty := true-- for _, b := range buf {- if _, err := bufStr.WriteString(b.GetText()); err != nil {- return nil, err- }- if b.GetChannel() != antlr.TokenHiddenChannel {- empty = false- }- }- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)-- // For the End position, use the current token (semicolon or EOF)- // instead of the last token in buf- sqls = append(sqls, base.SingleSQL{- Text: bufStr.String(),- BaseLine: buf[0].GetLine() - 1,- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{- Line: int32(token.GetLine()),- Column: int32(token.GetColumn()),- }, statement),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),- Empty: empty,- })- buf = nil- continue- }- }- return sqls, nil+ return base.SplitSQLByLexer(stream, cql.CqlLexerSEMI, statement)}diff --git a/backend/plugin/parser/doris/split.go b/backend/plugin/parser/doris/split.goindex eff61176b7..bd772d1bf6 100644--- a/backend/plugin/parser/doris/split.go+++ b/backend/plugin/parser/doris/split.go@@ -1,12 +1,9 @@package dorisimport (- "strings"-"github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/doris"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -22,40 +19,5 @@ func SplitSQL(statement string) ([]base.SingleSQL, error) {stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()- tokens := stream.GetAllTokens()- var buf []antlr.Token- var sqls []base.SingleSQL-- for i, token := range tokens {- if i < len(tokens)-1 {- buf = append(buf, token)- }- if (token.GetTokenType() == parser.DorisSQLLexerSEMICOLON || i == len(tokens)-1) && len(buf) > 0 {- bufStr := new(strings.Builder)- empty := true-- for _, b := range buf {- if _, err := bufStr.WriteString(b.GetText()); err != nil {- return nil, err- }- if b.GetChannel() != antlr.TokenHiddenChannel {- empty = false- }- }- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)- sqls = append(sqls, base.SingleSQL{- Text: bufStr.String(),- BaseLine: buf[0].GetLine() - 1,- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{- Line: int32(buf[len(buf)-1].GetLine()),- Column: int32(buf[len(buf)-1].GetColumn()),- }, statement),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),- Empty: empty,- })- buf = nil- continue- }- }- return sqls, nil+ return base.SplitSQLByLexer(stream, parser.DorisSQLLexerSEMICOLON, statement)}diff --git a/backend/plugin/parser/partiql/split.go b/backend/plugin/parser/partiql/split.goindex 75b81e1352..83685685de 100644--- a/backend/plugin/parser/partiql/split.go+++ b/backend/plugin/parser/partiql/split.go@@ -1,12 +1,9 @@package partiqlimport (- "strings"-"github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/partiql"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -21,43 +18,5 @@ func SplitSQL(statement string) ([]base.SingleSQL, error) {stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()- tokens := stream.GetAllTokens()- var buf []antlr.Token- var sqls []base.SingleSQL-- for i, token := range tokens {- if i < len(tokens)-1 {- buf = append(buf, token)- }- if (token.GetTokenType() == parser.PartiQLLexerCOLON_SEMI || i == len(tokens)-1) && len(buf) > 0 {- bufStr := new(strings.Builder)- empty := true-- for _, b := range buf {- if _, err := bufStr.WriteString(b.GetText()); err != nil {- return nil, err- }- if b.GetChannel() != antlr.TokenHiddenChannel {- empty = false- }- }- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)-- // For the End position, use the current token (semicolon or EOF)- // instead of the last token in buf- sqls = append(sqls, base.SingleSQL{- Text: bufStr.String(),- BaseLine: buf[0].GetLine() - 1,- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{- Line: int32(token.GetLine()),- Column: int32(token.GetColumn()),- }, statement),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),- Empty: empty,- })- buf = nil- continue- }- }- return sqls, nil+ return base.SplitSQLByLexer(stream, parser.PartiQLLexerCOLON_SEMI, statement)}diff --git a/backend/plugin/parser/snowflake/split.go b/backend/plugin/parser/snowflake/split.goindex 601991eb38..ad18496d95 100644--- a/backend/plugin/parser/snowflake/split.go+++ b/backend/plugin/parser/snowflake/split.go@@ -1,12 +1,9 @@package snowflakeimport (- "strings"-"github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/snowflake"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -21,43 +18,5 @@ func SplitSQL(statement string) ([]base.SingleSQL, error) {stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()- tokens := stream.GetAllTokens()- var buf []antlr.Token- var sqls []base.SingleSQL-- for i, token := range tokens {- if i < len(tokens)-1 {- buf = append(buf, token)- }- if (token.GetTokenType() == parser.SnowflakeLexerSEMI || i == len(tokens)-1) && len(buf) > 0 {- bufStr := new(strings.Builder)- empty := true-- for _, b := range buf {- if _, err := bufStr.WriteString(b.GetText()); err != nil {- return nil, err- }- if b.GetChannel() != antlr.TokenHiddenChannel {- empty = false- }- }- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)-- // For the End position, use the current token (semicolon or EOF)- // instead of the last token in buf- sqls = append(sqls, base.SingleSQL{- Text: bufStr.String(),- BaseLine: buf[0].GetLine() - 1,- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{- Line: int32(token.GetLine()),- Column: int32(token.GetColumn()),- }, statement),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),- Empty: empty,- })- buf = nil- continue- }- }- return sqls, nil+ return base.SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI, statement)}