Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(parser/partiql): change ParsePartiQL to return list of AST nodes (#18044)

h3n4l6c1db22315ed
diff --git a/backend/plugin/parser/partiql/partiql.go b/backend/plugin/parser/partiql/partiql.go
index 25fb2bb39c..5ab268e818 100644
--- a/backend/plugin/parser/partiql/partiql.go
+++ b/backend/plugin/parser/partiql/partiql.go
@@ -17,25 +17,45 @@ func init() {
}
// parsePartiQLForRegistry is the ParseFunc for PartiQL.
-// Returns antlr.Tree on success.
+// Returns []*ParseResult on success.
func parsePartiQLForRegistry(statement string) (any, error) {
result, err := ParsePartiQL(statement)
if err != nil {
return nil, err
}
- if result == nil {
- return nil, nil
- }
- return result.Tree, nil
+ return result, nil
}
type ParseResult struct {
- Tree antlr.Tree
- Tokens *antlr.CommonTokenStream
+ Tree antlr.Tree
+ Tokens *antlr.CommonTokenStream
+ BaseLine int
+}
+
+// ParsePartiQL parses the given PartiQL statement by using antlr4. Returns a list of AST and token stream if no error.
+func ParsePartiQL(statement string) ([]*ParseResult, error) {
+ stmts, err := SplitSQL(statement)
+ if err != nil {
+ return nil, err
+ }
+
+ var result []*ParseResult
+ for _, stmt := range stmts {
+ if stmt.Empty {
+ continue
+ }
+
+ parseResult, err := parseSinglePartiQL(stmt.Text, stmt.BaseLine)
+ if err != nil {
+ return nil, err
+ }
+ result = append(result, parseResult)
+ }
+
+ return result, nil
}
-// ParsePartiQL parses the given PartiQL statement by using antlr4. Returns the AST and token stream if no error.
-func ParsePartiQL(statement string) (*ParseResult, error) {
+func parseSinglePartiQL(statement string, baseLine int) (*ParseResult, error) {
statement = strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon) + "\n;"
inputStream := antlr.NewInputStream(statement)
lexer := parser.NewPartiQLLexer(inputStream)
@@ -46,12 +66,14 @@ func ParsePartiQL(statement string) (*ParseResult, error) {
lexer.RemoveErrorListeners()
lexerErrorListener := &base.ParseErrorListener{
Statement: statement,
+ BaseLine: baseLine,
}
lexer.AddErrorListener(lexerErrorListener)
p.RemoveErrorListeners()
parserErrorListener := &base.ParseErrorListener{
Statement: statement,
+ BaseLine: baseLine,
}
p.AddErrorListener(parserErrorListener)
@@ -68,8 +90,9 @@ func ParsePartiQL(statement string) (*ParseResult, error) {
}
result := &ParseResult{
- Tree: tree,
- Tokens: stream,
+ Tree: tree,
+ Tokens: stream,
+ BaseLine: baseLine,
}
return result, nil
diff --git a/backend/plugin/parser/partiql/query.go b/backend/plugin/parser/partiql/query.go
index 6a5af39054..9fdcc217a8 100644
--- a/backend/plugin/parser/partiql/query.go
+++ b/backend/plugin/parser/partiql/query.go
@@ -13,19 +13,21 @@ func init() {
}
func validateQuery(statement string) (bool, bool, error) {
- parseResult, err := ParsePartiQL(statement)
+ parseResults, err := ParsePartiQL(statement)
if err != nil {
return false, false, err
}
- if parseResult == nil {
- return false, false, nil
- }
- l := &queryValidateListener{
- valid: true,
+ for _, parseResult := range parseResults {
+ l := &queryValidateListener{
+ valid: true,
+ }
+ antlr.ParseTreeWalkerDefault.Walk(l, parseResult.Tree)
+ if !l.valid {
+ return false, false, nil
+ }
}
- antlr.ParseTreeWalkerDefault.Walk(l, parseResult.Tree)
- return l.valid, l.valid, nil
+ return true, true, nil
}
type queryValidateListener struct {
diff --git a/backend/plugin/parser/partiql/split.go b/backend/plugin/parser/partiql/split.go
index 824da7fde5..75b81e1352 100644
--- a/backend/plugin/parser/partiql/split.go
+++ b/backend/plugin/parser/partiql/split.go
@@ -1,7 +1,7 @@
package partiql
import (
- "errors"
+ "strings"
"github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/parser/partiql"
@@ -15,69 +15,49 @@ func init() {
base.RegisterSplitterFunc(storepb.Engine_DYNAMODB, SplitSQL)
}
+// SplitSQL splits the input into multiple SQL statements using semicolon as delimiter.
func SplitSQL(statement string) ([]base.SingleSQL, error) {
lexer := parser.NewPartiQLLexer(antlr.NewInputStream(statement))
- lexerErrorListener := &base.ParseErrorListener{
- Statement: statement,
- }
- lexer.RemoveErrorListeners()
- lexer.AddErrorListener(lexerErrorListener)
-
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
+ stream.Fill()
- p := parser.NewPartiQLParserParser(stream)
- parserErrorListener := &base.ParseErrorListener{
- Statement: statement,
- }
- p.RemoveErrorListeners()
- p.AddErrorListener(parserErrorListener)
-
- p.BuildParseTrees = true
-
- tree := p.Script()
- if lexerErrorListener.Err != nil {
- return nil, lexerErrorListener.Err
- }
-
- if parserErrorListener.Err != nil {
- return nil, parserErrorListener.Err
- }
-
- if tree == nil {
- return nil, errors.New("failed to split multiple statements")
- }
-
- var result []base.SingleSQL
tokens := stream.GetAllTokens()
-
- start := 0
- for _, semi := range tree.AllCOLON_SEMI() {
- pos := semi.GetSymbol().GetTokenIndex()
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
- result = append(result, base.SingleSQL{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
- BaseLine: tokens[start].GetLine() - 1,
- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()), Column: int32(tokens[pos].GetColumn())}, statement),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
- Empty: base.IsEmpty(tokens[start:pos+1], parser.PartiQLLexerCOLON_SEMI),
- })
- start = pos + 1
- }
- // For the last statement, it may not end with semicolon symbol, EOF symbol instead.
- eofPos := len(tokens) - 1
- if start < eofPos {
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
- result = append(result, base.SingleSQL{
- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
- BaseLine: tokens[start].GetLine() - 1,
- End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos].GetLine()), Column: int32(tokens[eofPos].GetColumn())}, statement),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
- Empty: base.IsEmpty(tokens[start:eofPos], parser.PartiQLLexerCOLON_SEMI),
- })
+ var buf []antlr.Token
+ var sqls []base.SingleSQL
+
+ for i, token := range tokens {
+ if i < len(tokens)-1 {
+ buf = append(buf, token)
+ }
+ if (token.GetTokenType() == parser.PartiQLLexerCOLON_SEMI || i == len(tokens)-1) && len(buf) > 0 {
+ bufStr := new(strings.Builder)
+ empty := true
+
+ for _, b := range buf {
+ if _, err := bufStr.WriteString(b.GetText()); err != nil {
+ return nil, err
+ }
+ if b.GetChannel() != antlr.TokenHiddenChannel {
+ empty = false
+ }
+ }
+ antlrPosition := base.FirstDefaultChannelTokenPosition(buf)
+
+ // For the End position, use the current token (semicolon or EOF)
+ // instead of the last token in buf
+ sqls = append(sqls, base.SingleSQL{
+ Text: bufStr.String(),
+ BaseLine: buf[0].GetLine() - 1,
+ End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
+ Line: int32(token.GetLine()),
+ Column: int32(token.GetColumn()),
+ }, statement),
+ Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Empty: empty,
+ })
+ buf = nil
+ continue
+ }
}
- return result, nil
+ return sqls, nil
}