Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(parser/doris): change ParseDorisSQL to return list of AST nodes (#18043)

h3n4lacaeaaa12bda
diff --git a/backend/plugin/parser/doris/doris.go b/backend/plugin/parser/doris/doris.go
index a981d56924..067b17e24f 100644
--- a/backend/plugin/parser/doris/doris.go
+++ b/backend/plugin/parser/doris/doris.go
@@ -13,7 +13,7 @@ func init() {
}
// parseDorisForRegistry is the ParseFunc for Doris.
-// Returns *ParseResult on success.
+// Returns []*ParseResult on success.
func parseDorisForRegistry(statement string) (any, error) {
result, err := ParseDorisSQL(statement)
if err != nil {
@@ -22,25 +22,50 @@ func parseDorisForRegistry(statement string) (any, error) {
return result, nil
}
-// ParseResult is the result of parsing a MySQL statement.
+// ParseResult is the result of parsing a Doris statement.
type ParseResult struct {
Tree antlr.Tree
Tokens *antlr.CommonTokenStream
BaseLine int
}
-func ParseDorisSQL(statement string) (*ParseResult, error) {
+// ParseDorisSQL parses the given SQL statement by using antlr4. Returns a list of AST and token stream if no error.
+func ParseDorisSQL(statement string) ([]*ParseResult, error) {
+ stmts, err := SplitSQL(statement)
+ if err != nil {
+ return nil, err
+ }
+
+ var result []*ParseResult
+ for _, stmt := range stmts {
+ if stmt.Empty {
+ continue
+ }
+
+ parseResult, err := parseSingleDorisSQL(stmt.Text, stmt.BaseLine)
+ if err != nil {
+ return nil, err
+ }
+ result = append(result, parseResult)
+ }
+
+ return result, nil
+}
+
+func parseSingleDorisSQL(statement string, baseLine int) (*ParseResult, error) {
lexer := parser.NewDorisSQLLexer(antlr.NewInputStream(statement))
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
p := parser.NewDorisSQLParser(stream)
lexerErrorListener := &base.ParseErrorListener{
Statement: statement,
+ BaseLine: baseLine,
}
lexer.RemoveErrorListeners()
lexer.AddErrorListener(lexerErrorListener)
parserErrorListener := &base.ParseErrorListener{
Statement: statement,
+ BaseLine: baseLine,
}
p.RemoveErrorListeners()
p.AddErrorListener(parserErrorListener)
@@ -57,8 +82,9 @@ func ParseDorisSQL(statement string) (*ParseResult, error) {
}
result := &ParseResult{
- Tree: tree,
- Tokens: stream,
+ Tree: tree,
+ Tokens: stream,
+ BaseLine: baseLine,
}
return result, nil
diff --git a/backend/plugin/parser/doris/doris_test.go b/backend/plugin/parser/doris/doris_test.go
index e89d3e1c48..6bd7d92267 100644
--- a/backend/plugin/parser/doris/doris_test.go
+++ b/backend/plugin/parser/doris/doris_test.go
@@ -26,8 +26,8 @@ func TestDorisSQLParser(t *testing.T) {
for _, test := range tests {
res, err := ParseDorisSQL(test.statement)
- if res != nil {
- _, ok := res.Tree.(*parser.SqlStatementsContext)
+ if len(res) > 0 {
+ _, ok := res[0].Tree.(*parser.SqlStatementsContext)
require.True(t, ok)
}
if test.errorMessage == "" {
diff --git a/backend/plugin/parser/doris/query.go b/backend/plugin/parser/doris/query.go
index 066da83e2d..f43b7c7607 100644
--- a/backend/plugin/parser/doris/query.go
+++ b/backend/plugin/parser/doris/query.go
@@ -15,16 +15,18 @@ func init() {
func validateQuery(statement string) (bool, bool, error) {
// TODO: support other readonly statements like SHOW TABLES, SHOW CREATE TABLE, etc.
- result, err := ParseDorisSQL(statement)
+ results, err := ParseDorisSQL(statement)
if err != nil {
return false, false, err
}
- l := &queryValidateListener{
- valid: true,
- }
- antlr.ParseTreeWalkerDefault.Walk(l, result.Tree)
- if !l.valid {
- return false, false, nil
+ for _, result := range results {
+ l := &queryValidateListener{
+ valid: true,
+ }
+ antlr.ParseTreeWalkerDefault.Walk(l, result.Tree)
+ if !l.valid {
+ return false, false, nil
+ }
}
return true, true, nil
}
diff --git a/backend/plugin/parser/doris/query_span_extractor.go b/backend/plugin/parser/doris/query_span_extractor.go
index f93f142ca3..6fcec909f3 100644
--- a/backend/plugin/parser/doris/query_span_extractor.go
+++ b/backend/plugin/parser/doris/query_span_extractor.go
@@ -6,6 +6,7 @@ import (
"github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/parser/doris"
+ "github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -30,18 +31,22 @@ func newQuerySpanExtractor(database string, gCtx base.GetQuerySpanContext, ignor
func (q *querySpanExtractor) getQuerySpan(ctx context.Context, statement string) (*base.QuerySpan, error) {
q.ctx = ctx
- parseResult, err := ParseDorisSQL(statement)
+ parseResults, err := ParseDorisSQL(statement)
if err != nil {
return nil, err
}
- if parseResult == nil {
+ if len(parseResults) == 0 {
return &base.QuerySpan{
SourceColumns: base.SourceColumnSet{},
Results: []base.QuerySpanResult{},
}, nil
}
+ if len(parseResults) != 1 {
+ return nil, errors.Errorf("expecting only one statement to get query span, but got %d", len(parseResults))
+ }
+ parseResult := parseResults[0]
accessTables := getAccessTables(q.defaultDatabase, parseResult, q.ctes, q.gCtx, q.ignoreCaseSensitive)
// We do not support simultaneous access to the system table and the user table
diff --git a/backend/plugin/parser/doris/split.go b/backend/plugin/parser/doris/split.go
index efb63134ed..eff61176b7 100644
--- a/backend/plugin/parser/doris/split.go
+++ b/backend/plugin/parser/doris/split.go
@@ -1,8 +1,10 @@
package doris
import (
+ "strings"
+
+ "github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/parser/doris"
- "github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -14,43 +16,46 @@ func init() {
base.RegisterSplitterFunc(storepb.Engine_DORIS, SplitSQL)
}
+// SplitSQL splits the input into multiple SQL statements using semicolon as delimiter.
func SplitSQL(statement string) ([]base.SingleSQL, error) {
- parseResult, err := ParseDorisSQL(statement)
- if err != nil {
- return nil, err
- }
+ lexer := parser.NewDorisSQLLexer(antlr.NewInputStream(statement))
+ stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
+ stream.Fill()
- var result []base.SingleSQL
- tree, ok := parseResult.Tree.(*parser.SqlStatementsContext)
- if !ok {
- return nil, errors.Errorf("failed to cast tree to SqlStatementsContext")
- }
- tokens := parseResult.Tokens.GetAllTokens()
- stream := parseResult.Tokens
-
- start := 0
- for _, singleStatement := range tree.AllSingleStatement() {
- var pos int
- if singleStatement.SEMICOLON() != nil {
- pos = singleStatement.SEMICOLON().GetSymbol().GetTokenIndex()
- } else {
- pos = singleStatement.GetStop().GetTokenIndex()
+ tokens := stream.GetAllTokens()
+ var buf []antlr.Token
+ var sqls []base.SingleSQL
+
+ for i, token := range tokens {
+ if i < len(tokens)-1 {
+ buf = append(buf, token)
}
+ if (token.GetTokenType() == parser.DorisSQLLexerSEMICOLON || i == len(tokens)-1) && len(buf) > 0 {
+ bufStr := new(strings.Builder)
+ empty := true
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
- result = append(result, base.SingleSQL{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
- BaseLine: tokens[start].GetLine() - 1,
- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
- Line: int32(tokens[pos].GetLine()),
- Column: int32(tokens[pos].GetColumn()),
- }, statement),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
- Empty: singleStatement.Statement() == nil,
- })
- start = pos + 1
+ for _, b := range buf {
+ if _, err := bufStr.WriteString(b.GetText()); err != nil {
+ return nil, err
+ }
+ if b.GetChannel() != antlr.TokenHiddenChannel {
+ empty = false
+ }
+ }
+ antlrPosition := base.FirstDefaultChannelTokenPosition(buf)
+ sqls = append(sqls, base.SingleSQL{
+ Text: bufStr.String(),
+ BaseLine: buf[0].GetLine() - 1,
+ End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
+ Line: int32(buf[len(buf)-1].GetLine()),
+ Column: int32(buf[len(buf)-1].GetColumn()),
+ }, statement),
+ Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Empty: empty,
+ })
+ buf = nil
+ continue
+ }
}
- return result, nil
+ return sqls, nil
}