Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser/doris): change ParseDorisSQL to return list of AST nodes (#18043)
h3n4lacaeaaa12bda
diff --git a/backend/plugin/parser/doris/doris.go b/backend/plugin/parser/doris/doris.goindex a981d56924..067b17e24f 100644--- a/backend/plugin/parser/doris/doris.go+++ b/backend/plugin/parser/doris/doris.go@@ -13,7 +13,7 @@ func init() {}// parseDorisForRegistry is the ParseFunc for Doris.-// Returns *ParseResult on success.+// Returns []*ParseResult on success.func parseDorisForRegistry(statement string) (any, error) {result, err := ParseDorisSQL(statement)if err != nil {@@ -22,25 +22,50 @@ func parseDorisForRegistry(statement string) (any, error) {return result, nil}-// ParseResult is the result of parsing a MySQL statement.+// ParseResult is the result of parsing a Doris statement.type ParseResult struct {Tree antlr.TreeTokens *antlr.CommonTokenStreamBaseLine int}-func ParseDorisSQL(statement string) (*ParseResult, error) {+// ParseDorisSQL parses the given SQL statement by using antlr4. Returns a list of AST and token stream if no error.+func ParseDorisSQL(statement string) ([]*ParseResult, error) {+ stmts, err := SplitSQL(statement)+ if err != nil {+ return nil, err+ }++ var result []*ParseResult+ for _, stmt := range stmts {+ if stmt.Empty {+ continue+ }++ parseResult, err := parseSingleDorisSQL(stmt.Text, stmt.BaseLine)+ if err != nil {+ return nil, err+ }+ result = append(result, parseResult)+ }++ return result, nil+}++func parseSingleDorisSQL(statement string, baseLine int) (*ParseResult, error) {lexer := parser.NewDorisSQLLexer(antlr.NewInputStream(statement))stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)p := parser.NewDorisSQLParser(stream)lexerErrorListener := &base.ParseErrorListener{Statement: statement,+ BaseLine: baseLine,}lexer.RemoveErrorListeners()lexer.AddErrorListener(lexerErrorListener)parserErrorListener := &base.ParseErrorListener{Statement: statement,+ BaseLine: baseLine,}p.RemoveErrorListeners()p.AddErrorListener(parserErrorListener)@@ -57,8 +82,9 @@ func ParseDorisSQL(statement string) (*ParseResult, error) {}result := &ParseResult{- Tree: tree,- Tokens: stream,+ Tree: tree,+ Tokens: stream,+ BaseLine: baseLine,}return result, nildiff --git a/backend/plugin/parser/doris/doris_test.go b/backend/plugin/parser/doris/doris_test.goindex e89d3e1c48..6bd7d92267 100644--- a/backend/plugin/parser/doris/doris_test.go+++ b/backend/plugin/parser/doris/doris_test.go@@ -26,8 +26,8 @@ func TestDorisSQLParser(t *testing.T) {for _, test := range tests {res, err := ParseDorisSQL(test.statement)- if res != nil {- _, ok := res.Tree.(*parser.SqlStatementsContext)+ if len(res) > 0 {+ _, ok := res[0].Tree.(*parser.SqlStatementsContext)require.True(t, ok)}if test.errorMessage == "" {diff --git a/backend/plugin/parser/doris/query.go b/backend/plugin/parser/doris/query.goindex 066da83e2d..f43b7c7607 100644--- a/backend/plugin/parser/doris/query.go+++ b/backend/plugin/parser/doris/query.go@@ -15,16 +15,18 @@ func init() {func validateQuery(statement string) (bool, bool, error) {// TODO: support other readonly statements like SHOW TABLES, SHOW CREATE TABLE, etc.- result, err := ParseDorisSQL(statement)+ results, err := ParseDorisSQL(statement)if err != nil {return false, false, err}- l := &queryValidateListener{- valid: true,- }- antlr.ParseTreeWalkerDefault.Walk(l, result.Tree)- if !l.valid {- return false, false, nil+ for _, result := range results {+ l := &queryValidateListener{+ valid: true,+ }+ antlr.ParseTreeWalkerDefault.Walk(l, result.Tree)+ if !l.valid {+ return false, false, nil+ }}return true, true, nil}diff --git a/backend/plugin/parser/doris/query_span_extractor.go b/backend/plugin/parser/doris/query_span_extractor.goindex f93f142ca3..6fcec909f3 100644--- a/backend/plugin/parser/doris/query_span_extractor.go+++ b/backend/plugin/parser/doris/query_span_extractor.go@@ -6,6 +6,7 @@ import ("github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/doris"+ "github.com/pkg/errors""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -30,18 +31,22 @@ func newQuerySpanExtractor(database string, gCtx base.GetQuerySpanContext, ignorfunc (q *querySpanExtractor) getQuerySpan(ctx context.Context, statement string) (*base.QuerySpan, error) {q.ctx = ctx- parseResult, err := ParseDorisSQL(statement)+ parseResults, err := ParseDorisSQL(statement)if err != nil {return nil, err}- if parseResult == nil {+ if len(parseResults) == 0 {return &base.QuerySpan{SourceColumns: base.SourceColumnSet{},Results: []base.QuerySpanResult{},}, nil}+ if len(parseResults) != 1 {+ return nil, errors.Errorf("expecting only one statement to get query span, but got %d", len(parseResults))+ }+ parseResult := parseResults[0]accessTables := getAccessTables(q.defaultDatabase, parseResult, q.ctes, q.gCtx, q.ignoreCaseSensitive)// We do not support simultaneous access to the system table and the user tablediff --git a/backend/plugin/parser/doris/split.go b/backend/plugin/parser/doris/split.goindex efb63134ed..eff61176b7 100644--- a/backend/plugin/parser/doris/split.go+++ b/backend/plugin/parser/doris/split.go@@ -1,8 +1,10 @@package dorisimport (+ "strings"++ "github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/doris"- "github.com/pkg/errors""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -14,43 +16,46 @@ func init() {base.RegisterSplitterFunc(storepb.Engine_DORIS, SplitSQL)}+// SplitSQL splits the input into multiple SQL statements using semicolon as delimiter.func SplitSQL(statement string) ([]base.SingleSQL, error) {- parseResult, err := ParseDorisSQL(statement)- if err != nil {- return nil, err- }+ lexer := parser.NewDorisSQLLexer(antlr.NewInputStream(statement))+ stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)+ stream.Fill()- var result []base.SingleSQL- tree, ok := parseResult.Tree.(*parser.SqlStatementsContext)- if !ok {- return nil, errors.Errorf("failed to cast tree to SqlStatementsContext")- }- tokens := parseResult.Tokens.GetAllTokens()- stream := parseResult.Tokens-- start := 0- for _, singleStatement := range tree.AllSingleStatement() {- var pos int- if singleStatement.SEMICOLON() != nil {- pos = singleStatement.SEMICOLON().GetSymbol().GetTokenIndex()- } else {- pos = singleStatement.GetStop().GetTokenIndex()+ tokens := stream.GetAllTokens()+ var buf []antlr.Token+ var sqls []base.SingleSQL++ for i, token := range tokens {+ if i < len(tokens)-1 {+ buf = append(buf, token)}+ if (token.GetTokenType() == parser.DorisSQLLexerSEMICOLON || i == len(tokens)-1) && len(buf) > 0 {+ bufStr := new(strings.Builder)+ empty := true- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.- result = append(result, base.SingleSQL{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),- BaseLine: tokens[start].GetLine() - 1,- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{- Line: int32(tokens[pos].GetLine()),- Column: int32(tokens[pos].GetColumn()),- }, statement),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),- Empty: singleStatement.Statement() == nil,- })- start = pos + 1+ for _, b := range buf {+ if _, err := bufStr.WriteString(b.GetText()); err != nil {+ return nil, err+ }+ if b.GetChannel() != antlr.TokenHiddenChannel {+ empty = false+ }+ }+ antlrPosition := base.FirstDefaultChannelTokenPosition(buf)+ sqls = append(sqls, base.SingleSQL{+ Text: bufStr.String(),+ BaseLine: buf[0].GetLine() - 1,+ End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{+ Line: int32(buf[len(buf)-1].GetLine()),+ Column: int32(buf[len(buf)-1].GetColumn()),+ }, statement),+ Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Empty: empty,+ })+ buf = nil+ continue+ }}- return result, nil+ return sqls, nil}