Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: create grammar-free SplitSQLByLexer helper for ANTLR-based splitters (#18158)

h3n4l2bc565ada160
diff --git a/backend/plugin/parser/base/lexer.go b/backend/plugin/parser/base/lexer.go
index f602a45474..812d8e7536 100644
--- a/backend/plugin/parser/base/lexer.go
+++ b/backend/plugin/parser/base/lexer.go
@@ -1,6 +1,8 @@
package base
import (
+ "strings"
+
"github.com/antlr4-go/antlr/v4"
"github.com/bytebase/bytebase/backend/common"
@@ -53,3 +55,69 @@ func IsEmpty(tokens []antlr.Token, semi int) bool {
}
return true
}
+
+// SplitSQLByLexer is a grammar-free helper function that splits SQL statements using an ANTLR lexer.
+// It works with any ANTLR-based lexer by accepting the token stream and semicolon token type as parameters.
+//
+// Parameters:
+// - stream: The ANTLR token stream (must be already filled with stream.Fill())
+// - semiTokenType: The token type value for semicolon in the specific grammar
+// - statement: The original SQL statement string (used for position conversion)
+//
+// Returns:
+// - A slice of SingleSQL, each representing one statement with its text, position, and metadata
+//
+// Example usage:
+//
+// lexer := parser.NewSnowflakeLexer(antlr.NewInputStream(statement))
+// stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
+// stream.Fill()
+// return SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI, statement)
+func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, statement string) ([]SingleSQL, error) {
+ tokens := stream.GetAllTokens()
+ var buf []antlr.Token
+ var sqls []SingleSQL
+
+ for i, token := range tokens {
+ // Collect all tokens except the last one (EOF)
+ if i < len(tokens)-1 {
+ buf = append(buf, token)
+ }
+
+ // Split on semicolon or at the end of tokens
+ if (token.GetTokenType() == semiTokenType || i == len(tokens)-1) && len(buf) > 0 {
+ // Reconstruct the text from tokens
+ bufStr := new(strings.Builder)
+ empty := true
+
+ for _, b := range buf {
+ if _, err := bufStr.WriteString(b.GetText()); err != nil {
+ return nil, err
+ }
+ // Check if there's any non-hidden token (actual code, not just comments/whitespace)
+ if b.GetChannel() != antlr.TokenHiddenChannel {
+ empty = false
+ }
+ }
+
+ // Get the position of the first default channel token for Start position
+ antlrPosition := FirstDefaultChannelTokenPosition(buf)
+
+ sqls = append(sqls, SingleSQL{
+ Text: bufStr.String(),
+ BaseLine: buf[0].GetLine() - 1, // BaseLine is the offset of the first token
+ End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
+ Line: int32(token.GetLine()),
+ Column: int32(token.GetColumn()),
+ }, statement),
+ Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Empty: empty,
+ })
+
+ buf = nil
+ continue
+ }
+ }
+
+ return sqls, nil
+}
diff --git a/backend/plugin/parser/cassandra/split.go b/backend/plugin/parser/cassandra/split.go
index 2a44a40467..a3517e251f 100644
--- a/backend/plugin/parser/cassandra/split.go
+++ b/backend/plugin/parser/cassandra/split.go
@@ -1,12 +1,9 @@
package cassandra
import (
- "strings"
-
"github.com/antlr4-go/antlr/v4"
"github.com/bytebase/parser/cql"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -21,43 +18,5 @@ func SplitSQL(statement string) ([]base.SingleSQL, error) {
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
- tokens := stream.GetAllTokens()
- var buf []antlr.Token
- var sqls []base.SingleSQL
-
- for i, token := range tokens {
- if i < len(tokens)-1 {
- buf = append(buf, token)
- }
- if (token.GetTokenType() == cql.CqlLexerSEMI || i == len(tokens)-1) && len(buf) > 0 {
- bufStr := new(strings.Builder)
- empty := true
-
- for _, b := range buf {
- if _, err := bufStr.WriteString(b.GetText()); err != nil {
- return nil, err
- }
- if b.GetChannel() != antlr.TokenHiddenChannel {
- empty = false
- }
- }
- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)
-
- // For the End position, use the current token (semicolon or EOF)
- // instead of the last token in buf
- sqls = append(sqls, base.SingleSQL{
- Text: bufStr.String(),
- BaseLine: buf[0].GetLine() - 1,
- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
- Line: int32(token.GetLine()),
- Column: int32(token.GetColumn()),
- }, statement),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
- Empty: empty,
- })
- buf = nil
- continue
- }
- }
- return sqls, nil
+ return base.SplitSQLByLexer(stream, cql.CqlLexerSEMI, statement)
}
diff --git a/backend/plugin/parser/doris/split.go b/backend/plugin/parser/doris/split.go
index eff61176b7..bd772d1bf6 100644
--- a/backend/plugin/parser/doris/split.go
+++ b/backend/plugin/parser/doris/split.go
@@ -1,12 +1,9 @@
package doris
import (
- "strings"
-
"github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/parser/doris"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -22,40 +19,5 @@ func SplitSQL(statement string) ([]base.SingleSQL, error) {
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
- tokens := stream.GetAllTokens()
- var buf []antlr.Token
- var sqls []base.SingleSQL
-
- for i, token := range tokens {
- if i < len(tokens)-1 {
- buf = append(buf, token)
- }
- if (token.GetTokenType() == parser.DorisSQLLexerSEMICOLON || i == len(tokens)-1) && len(buf) > 0 {
- bufStr := new(strings.Builder)
- empty := true
-
- for _, b := range buf {
- if _, err := bufStr.WriteString(b.GetText()); err != nil {
- return nil, err
- }
- if b.GetChannel() != antlr.TokenHiddenChannel {
- empty = false
- }
- }
- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)
- sqls = append(sqls, base.SingleSQL{
- Text: bufStr.String(),
- BaseLine: buf[0].GetLine() - 1,
- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
- Line: int32(buf[len(buf)-1].GetLine()),
- Column: int32(buf[len(buf)-1].GetColumn()),
- }, statement),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
- Empty: empty,
- })
- buf = nil
- continue
- }
- }
- return sqls, nil
+ return base.SplitSQLByLexer(stream, parser.DorisSQLLexerSEMICOLON, statement)
}
diff --git a/backend/plugin/parser/partiql/split.go b/backend/plugin/parser/partiql/split.go
index 75b81e1352..83685685de 100644
--- a/backend/plugin/parser/partiql/split.go
+++ b/backend/plugin/parser/partiql/split.go
@@ -1,12 +1,9 @@
package partiql
import (
- "strings"
-
"github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/parser/partiql"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -21,43 +18,5 @@ func SplitSQL(statement string) ([]base.SingleSQL, error) {
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
- tokens := stream.GetAllTokens()
- var buf []antlr.Token
- var sqls []base.SingleSQL
-
- for i, token := range tokens {
- if i < len(tokens)-1 {
- buf = append(buf, token)
- }
- if (token.GetTokenType() == parser.PartiQLLexerCOLON_SEMI || i == len(tokens)-1) && len(buf) > 0 {
- bufStr := new(strings.Builder)
- empty := true
-
- for _, b := range buf {
- if _, err := bufStr.WriteString(b.GetText()); err != nil {
- return nil, err
- }
- if b.GetChannel() != antlr.TokenHiddenChannel {
- empty = false
- }
- }
- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)
-
- // For the End position, use the current token (semicolon or EOF)
- // instead of the last token in buf
- sqls = append(sqls, base.SingleSQL{
- Text: bufStr.String(),
- BaseLine: buf[0].GetLine() - 1,
- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
- Line: int32(token.GetLine()),
- Column: int32(token.GetColumn()),
- }, statement),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
- Empty: empty,
- })
- buf = nil
- continue
- }
- }
- return sqls, nil
+ return base.SplitSQLByLexer(stream, parser.PartiQLLexerCOLON_SEMI, statement)
}
diff --git a/backend/plugin/parser/snowflake/split.go b/backend/plugin/parser/snowflake/split.go
index 601991eb38..ad18496d95 100644
--- a/backend/plugin/parser/snowflake/split.go
+++ b/backend/plugin/parser/snowflake/split.go
@@ -1,12 +1,9 @@
package snowflake
import (
- "strings"
-
"github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/parser/snowflake"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -21,43 +18,5 @@ func SplitSQL(statement string) ([]base.SingleSQL, error) {
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
- tokens := stream.GetAllTokens()
- var buf []antlr.Token
- var sqls []base.SingleSQL
-
- for i, token := range tokens {
- if i < len(tokens)-1 {
- buf = append(buf, token)
- }
- if (token.GetTokenType() == parser.SnowflakeLexerSEMI || i == len(tokens)-1) && len(buf) > 0 {
- bufStr := new(strings.Builder)
- empty := true
-
- for _, b := range buf {
- if _, err := bufStr.WriteString(b.GetText()); err != nil {
- return nil, err
- }
- if b.GetChannel() != antlr.TokenHiddenChannel {
- empty = false
- }
- }
- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)
-
- // For the End position, use the current token (semicolon or EOF)
- // instead of the last token in buf
- sqls = append(sqls, base.SingleSQL{
- Text: bufStr.String(),
- BaseLine: buf[0].GetLine() - 1,
- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
- Line: int32(token.GetLine()),
- Column: int32(token.GetColumn()),
- }, statement),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
- Empty: empty,
- })
- buf = nil
- continue
- }
- }
- return sqls, nil
+ return base.SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI, statement)
}