Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: standardize Spanner parser to return list of ParseResults (#18163)
h3n4l324acb9688bd
diff --git a/backend/plugin/parser/spanner/query_span_extractor.go b/backend/plugin/parser/spanner/query_span_extractor.goindex b53043cc3f..86738e9c73 100644--- a/backend/plugin/parser/spanner/query_span_extractor.go+++ b/backend/plugin/parser/spanner/query_span_extractor.go@@ -43,7 +43,13 @@ func (q *querySpanExtractor) getQuerySpan(ctx context.Context, stmt string) (*baif err != nil {return nil, err}- tree := parseResults.Tree++ if len(parseResults) != 1 {+ return nil, errors.Errorf("expected exactly 1 statement, got %d", len(parseResults))+ }++ parseResult := parseResults[0]+ tree := parseResult.Treeq.ctx = ctxaccessTables, err := getAccessTables(q.defaultDatabase, tree)if err != nil {diff --git a/backend/plugin/parser/spanner/spanner.go b/backend/plugin/parser/spanner/spanner.goindex 95eafb8a78..75c1b5af85 100644--- a/backend/plugin/parser/spanner/spanner.go+++ b/backend/plugin/parser/spanner/spanner.go@@ -12,12 +12,37 @@ import ()type ParseResult struct {- Tree antlr.Tree- Tokens *antlr.CommonTokenStream+ Tree antlr.Tree+ Tokens *antlr.CommonTokenStream+ BaseLine int}-// ParseSpannerGoogleSQL parses the given SQL statement by using antlr4. Returns the AST and token stream if no error.-func ParseSpannerGoogleSQL(statement string) (*ParseResult, error) {+// ParseSpannerGoogleSQL parses the given SQL and returns a list of ParseResult (one per statement).+// Use the GoogleSQL parser based on antlr4.+func ParseSpannerGoogleSQL(sql string) ([]*ParseResult, error) {+ stmts, err := SplitSQL(sql)+ if err != nil {+ return nil, err+ }++ var results []*ParseResult+ for _, stmt := range stmts {+ if stmt.Empty {+ continue+ }++ parseResult, err := parseSingleSpannerGoogleSQL(stmt.Text, stmt.BaseLine)+ if err != nil {+ return nil, err+ }+ results = append(results, parseResult)+ }++ return results, nil+}++// parseSingleSpannerGoogleSQL parses a single Spanner statement and returns the ParseResult.+func parseSingleSpannerGoogleSQL(statement string, baseLine int) (*ParseResult, error) {statement = strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon) + "\n;"inputStream := antlr.NewInputStream(statement)lexer := parser.NewGoogleSQLLexer(inputStream)@@ -28,12 +53,14 @@ func ParseSpannerGoogleSQL(statement string) (*ParseResult, error) {lexer.RemoveErrorListeners()lexerErrorListener := &base.ParseErrorListener{Statement: statement,+ BaseLine: baseLine,}lexer.AddErrorListener(lexerErrorListener)p.RemoveErrorListeners()parserErrorListener := &base.ParseErrorListener{Statement: statement,+ BaseLine: baseLine,}p.AddErrorListener(parserErrorListener)@@ -50,8 +77,9 @@ func ParseSpannerGoogleSQL(statement string) (*ParseResult, error) {}result := &ParseResult{- Tree: tree,- Tokens: stream,+ Tree: tree,+ Tokens: stream,+ BaseLine: baseLine,}return result, nildiff --git a/backend/plugin/parser/spanner/split.go b/backend/plugin/parser/spanner/split.gonew file mode 100644index 0000000000..e3cf19f4c7--- /dev/null+++ b/backend/plugin/parser/spanner/split.go@@ -0,0 +1,106 @@+package spanner++import (+ "github.com/antlr4-go/antlr/v4"+ parser "github.com/bytebase/parser/googlesql"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterSplitterFunc(storepb.Engine_SPANNER, SplitSQL)+}++// SplitSQL splits the given SQL statement into multiple SQL statements using ANTLR parser.+// This handles BEGIN/END blocks, CASE, IF, LOOP, etc. correctly.+func SplitSQL(statement string) ([]base.SingleSQL, error) {+ lexer := parser.NewGoogleSQLLexer(antlr.NewInputStream(statement))+ stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)++ p := parser.NewGoogleSQLParser(stream)+ lexerErrorListener := &base.ParseErrorListener{+ Statement: statement,+ }+ lexer.RemoveErrorListeners()+ lexer.AddErrorListener(lexerErrorListener)++ parserErrorListener := &base.ParseErrorListener{+ Statement: statement,+ }+ p.RemoveErrorListeners()+ p.AddErrorListener(parserErrorListener)++ p.BuildParseTrees = true++ tree := p.Root()+ if lexerErrorListener.Err != nil {+ return nil, lexerErrorListener.Err+ }++ if parserErrorListener.Err != nil {+ return nil, parserErrorListener.Err+ }++ if tree == nil || tree.Stmts() == nil {+ return nil, errors.New("failed to split multiple statements")+ }++ var result []base.SingleSQL+ tokens := stream.GetAllTokens()++ // Get all statement-terminating semicolons from the parse tree+ allStatements := tree.Stmts().AllUnterminated_sql_statement()+ if len(allStatements) == 0 {+ return result, nil+ }++ start := 0+ for i, stmt := range allStatements {+ // Find the semicolon after this statement+ var endPos int+ if i < len(allStatements)-1 {+ // Not the last statement - find the semicolon between this and next statement+ nextStmt := allStatements[i+1]+ nextStmtStart := nextStmt.GetStart().GetTokenIndex()+ // Find the semicolon just before the next statement+ endPos = nextStmtStart - 1+ // Skip back to find the actual semicolon+ for endPos >= start && tokens[endPos].GetTokenType() != parser.GoogleSQLLexerSEMI_SYMBOL {+ endPos--+ }+ } else {+ // Last statement - may or may not have semicolon+ stmtStop := stmt.GetStop().GetTokenIndex()+ endPos = stmtStop+ // Check if there's a semicolon after the statement+ for endPos < len(tokens)-1 {+ endPos+++ if tokens[endPos].GetTokenType() == parser.GoogleSQLLexerSEMI_SYMBOL {+ break+ }+ }+ }++ if endPos < start {+ continue+ }++ antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : endPos+1])+ result = append(result, base.SingleSQL{+ Text: stream.GetTextFromTokens(tokens[start], tokens[endPos]),+ BaseLine: tokens[start].GetLine() - 1,+ End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{+ Line: int32(tokens[endPos].GetLine()),+ Column: int32(tokens[endPos].GetColumn()),+ }, statement),+ Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Empty: base.IsEmpty(tokens[start:endPos+1], parser.GoogleSQLLexerSEMI_SYMBOL),+ })+ start = endPos + 1+ }++ return result, nil+}diff --git a/backend/plugin/parser/spanner/split_begin_end_test.go b/backend/plugin/parser/spanner/split_begin_end_test.gonew file mode 100644index 0000000000..e76a4d5bba--- /dev/null+++ b/backend/plugin/parser/spanner/split_begin_end_test.go@@ -0,0 +1,32 @@+package spanner++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestSplitSQLWithBeginEnd(t *testing.T) {+ // Test with CASE expression (no semicolons inside CASE in SQL)+ statement := `SELECT+ CASE+ WHEN status = 'active' THEN 1+ WHEN status = 'inactive' THEN 0+ END+FROM users;+SELECT * FROM orders;`++ list, err := SplitSQL(statement)+ require.NoError(t, err)+ require.Equal(t, 2, len(list), "Should split into 2 statements")++ // First statement: SELECT with CASE+ require.Contains(t, list[0].Text, "SELECT")+ require.Contains(t, list[0].Text, "CASE")+ require.Contains(t, list[0].Text, "END")+ require.False(t, list[0].Empty)++ // Second statement: SELECT FROM orders+ require.Contains(t, list[1].Text, "SELECT * FROM orders")+ require.False(t, list[1].Empty)+}diff --git a/backend/plugin/parser/spanner/split_test.go b/backend/plugin/parser/spanner/split_test.gonew file mode 100644index 0000000000..4660d3ce32--- /dev/null+++ b/backend/plugin/parser/spanner/split_test.go@@ -0,0 +1,53 @@+package spanner++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestSplitSQL(t *testing.T) {+ statement := `SELECT * FROM users;+ SELECT * FROM orders;+ SELECT * FROM products;`+ want := []base.SingleSQL{+ {+ Text: "SELECT * FROM users;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 20},+ Empty: false,+ },+ {+ Text: "\n\tSELECT * FROM orders;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 2, Column: 2},+ End: &storepb.Position{Line: 2, Column: 22},+ Empty: false,+ },+ {+ Text: "\n\tSELECT * FROM products;",+ BaseLine: 1,+ Start: &storepb.Position{Line: 3, Column: 2},+ End: &storepb.Position{Line: 3, Column: 24},+ Empty: false,+ },+ }++ list, err := SplitSQL(statement)+ require.NoError(t, err)+ require.Equal(t, want, list)+}++func TestSplitSQLSingleStatement(t *testing.T) {+ statement := "SELECT * FROM users;"+ list, err := SplitSQL(statement)+ require.NoError(t, err)+ require.Equal(t, 1, len(list))+ require.Equal(t, "SELECT * FROM users;", list[0].Text)+ require.Equal(t, 0, list[0].BaseLine)+ require.False(t, list[0].Empty)+}