Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser/partiql): change ParsePartiQL to return list of AST nodes (#18044)
h3n4l6c1db22315ed
diff --git a/backend/plugin/parser/partiql/partiql.go b/backend/plugin/parser/partiql/partiql.goindex 25fb2bb39c..5ab268e818 100644--- a/backend/plugin/parser/partiql/partiql.go+++ b/backend/plugin/parser/partiql/partiql.go@@ -17,25 +17,45 @@ func init() {}// parsePartiQLForRegistry is the ParseFunc for PartiQL.-// Returns antlr.Tree on success.+// Returns []*ParseResult on success.func parsePartiQLForRegistry(statement string) (any, error) {result, err := ParsePartiQL(statement)if err != nil {return nil, err}- if result == nil {- return nil, nil- }- return result.Tree, nil+ return result, nil}type ParseResult struct {- Tree antlr.Tree- Tokens *antlr.CommonTokenStream+ Tree antlr.Tree+ Tokens *antlr.CommonTokenStream+ BaseLine int+}++// ParsePartiQL parses the given PartiQL statement by using antlr4. Returns a list of AST and token stream if no error.+func ParsePartiQL(statement string) ([]*ParseResult, error) {+ stmts, err := SplitSQL(statement)+ if err != nil {+ return nil, err+ }++ var result []*ParseResult+ for _, stmt := range stmts {+ if stmt.Empty {+ continue+ }++ parseResult, err := parseSinglePartiQL(stmt.Text, stmt.BaseLine)+ if err != nil {+ return nil, err+ }+ result = append(result, parseResult)+ }++ return result, nil}-// ParsePartiQL parses the given PartiQL statement by using antlr4. Returns the AST and token stream if no error.-func ParsePartiQL(statement string) (*ParseResult, error) {+func parseSinglePartiQL(statement string, baseLine int) (*ParseResult, error) {statement = strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon) + "\n;"inputStream := antlr.NewInputStream(statement)lexer := parser.NewPartiQLLexer(inputStream)@@ -46,12 +66,14 @@ func ParsePartiQL(statement string) (*ParseResult, error) {lexer.RemoveErrorListeners()lexerErrorListener := &base.ParseErrorListener{Statement: statement,+ BaseLine: baseLine,}lexer.AddErrorListener(lexerErrorListener)p.RemoveErrorListeners()parserErrorListener := &base.ParseErrorListener{Statement: statement,+ BaseLine: baseLine,}p.AddErrorListener(parserErrorListener)@@ -68,8 +90,9 @@ func ParsePartiQL(statement string) (*ParseResult, error) {}result := &ParseResult{- Tree: tree,- Tokens: stream,+ Tree: tree,+ Tokens: stream,+ BaseLine: baseLine,}return result, nildiff --git a/backend/plugin/parser/partiql/query.go b/backend/plugin/parser/partiql/query.goindex 6a5af39054..9fdcc217a8 100644--- a/backend/plugin/parser/partiql/query.go+++ b/backend/plugin/parser/partiql/query.go@@ -13,19 +13,21 @@ func init() {}func validateQuery(statement string) (bool, bool, error) {- parseResult, err := ParsePartiQL(statement)+ parseResults, err := ParsePartiQL(statement)if err != nil {return false, false, err}- if parseResult == nil {- return false, false, nil- }- l := &queryValidateListener{- valid: true,+ for _, parseResult := range parseResults {+ l := &queryValidateListener{+ valid: true,+ }+ antlr.ParseTreeWalkerDefault.Walk(l, parseResult.Tree)+ if !l.valid {+ return false, false, nil+ }}- antlr.ParseTreeWalkerDefault.Walk(l, parseResult.Tree)- return l.valid, l.valid, nil+ return true, true, nil}type queryValidateListener struct {diff --git a/backend/plugin/parser/partiql/split.go b/backend/plugin/parser/partiql/split.goindex 824da7fde5..75b81e1352 100644--- a/backend/plugin/parser/partiql/split.go+++ b/backend/plugin/parser/partiql/split.go@@ -1,7 +1,7 @@package partiqlimport (- "errors"+ "strings""github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/partiql"@@ -15,69 +15,49 @@ func init() {base.RegisterSplitterFunc(storepb.Engine_DYNAMODB, SplitSQL)}+// SplitSQL splits the input into multiple SQL statements using semicolon as delimiter.func SplitSQL(statement string) ([]base.SingleSQL, error) {lexer := parser.NewPartiQLLexer(antlr.NewInputStream(statement))- lexerErrorListener := &base.ParseErrorListener{- Statement: statement,- }- lexer.RemoveErrorListeners()- lexer.AddErrorListener(lexerErrorListener)-stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)+ stream.Fill()- p := parser.NewPartiQLParserParser(stream)- parserErrorListener := &base.ParseErrorListener{- Statement: statement,- }- p.RemoveErrorListeners()- p.AddErrorListener(parserErrorListener)-- p.BuildParseTrees = true-- tree := p.Script()- if lexerErrorListener.Err != nil {- return nil, lexerErrorListener.Err- }-- if parserErrorListener.Err != nil {- return nil, parserErrorListener.Err- }-- if tree == nil {- return nil, errors.New("failed to split multiple statements")- }-- var result []base.SingleSQLtokens := stream.GetAllTokens()-- start := 0- for _, semi := range tree.AllCOLON_SEMI() {- pos := semi.GetSymbol().GetTokenIndex()- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.- result = append(result, base.SingleSQL{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),- BaseLine: tokens[start].GetLine() - 1,- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()), Column: int32(tokens[pos].GetColumn())}, statement),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),- Empty: base.IsEmpty(tokens[start:pos+1], parser.PartiQLLexerCOLON_SEMI),- })- start = pos + 1- }- // For the last statement, it may not end with semicolon symbol, EOF symbol instead.- eofPos := len(tokens) - 1- if start < eofPos {- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.- result = append(result, base.SingleSQL{- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),- BaseLine: tokens[start].GetLine() - 1,- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos].GetLine()), Column: int32(tokens[eofPos].GetColumn())}, statement),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),- Empty: base.IsEmpty(tokens[start:eofPos], parser.PartiQLLexerCOLON_SEMI),- })+ var buf []antlr.Token+ var sqls []base.SingleSQL++ for i, token := range tokens {+ if i < len(tokens)-1 {+ buf = append(buf, token)+ }+ if (token.GetTokenType() == parser.PartiQLLexerCOLON_SEMI || i == len(tokens)-1) && len(buf) > 0 {+ bufStr := new(strings.Builder)+ empty := true++ for _, b := range buf {+ if _, err := bufStr.WriteString(b.GetText()); err != nil {+ return nil, err+ }+ if b.GetChannel() != antlr.TokenHiddenChannel {+ empty = false+ }+ }+ antlrPosition := base.FirstDefaultChannelTokenPosition(buf)++ // For the End position, use the current token (semicolon or EOF)+ // instead of the last token in buf+ sqls = append(sqls, base.SingleSQL{+ Text: bufStr.String(),+ BaseLine: buf[0].GetLine() - 1,+ End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{+ Line: int32(token.GetLine()),+ Column: int32(token.GetColumn()),+ }, statement),+ Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Empty: empty,+ })+ buf = nil+ continue+ }}- return result, nil+ return sqls, nil}