Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser/cosmosdb): standardize ParseCosmosDBQuery to return list of AST nodes (#18114)
h3n4l991605956962
diff --git a/backend/plugin/parser/cosmosdb/cosmosdb.go b/backend/plugin/parser/cosmosdb/cosmosdb.goindex 7606e6233d..d31da387b6 100644--- a/backend/plugin/parser/cosmosdb/cosmosdb.go+++ b/backend/plugin/parser/cosmosdb/cosmosdb.go@@ -1,37 +1,70 @@package cosmosdbimport (+ "strings"+"github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/cosmosdb""github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/utils")type ParseResult struct {- Tree antlr.Tree- Tokens *antlr.CommonTokenStream+ Tree antlr.Tree+ Tokens *antlr.CommonTokenStream+ BaseLine int}+// ParseCosmosDBQuery parses the given CosmosDB SQL statement by using antlr4. Returns a list of AST and token stream if no error.+// Note: CosmosDB only supports single SELECT statements, so this will typically return a list with one element.func ParseCosmosDBQuery(statement string) ([]*ParseResult, error) {- lexer := parser.NewCosmosDBLexer(antlr.NewInputStream(statement))+ stmts, err := SplitSQL(statement)+ if err != nil {+ return nil, err+ }++ var result []*ParseResult+ for _, stmt := range stmts {+ if stmt.Empty {+ continue+ }++ parseResult, err := parseSingleCosmosDBQuery(stmt.Text, stmt.BaseLine)+ if err != nil {+ return nil, err+ }+ result = append(result, parseResult)+ }++ return result, nil+}++func parseSingleCosmosDBQuery(statement string, baseLine int) (*ParseResult, error) {+ statement = strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon)+ inputStream := antlr.NewInputStream(statement)+ lexer := parser.NewCosmosDBLexer(inputStream)stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)p := parser.NewCosmosDBParser(stream)+ // Remove default error listener and add our own error listener.+ lexer.RemoveErrorListeners()lexerErrorListener := &base.ParseErrorListener{Statement: statement,+ BaseLine: baseLine,}- lexer.RemoveErrorListeners()lexer.AddErrorListener(lexerErrorListener)+ p.RemoveErrorListeners()parserErrorListener := &base.ParseErrorListener{Statement: statement,+ BaseLine: baseLine,}- p.RemoveErrorListeners()p.AddErrorListener(parserErrorListener)p.BuildParseTrees = true- root := p.Root()+ tree := p.Root()if lexerErrorListener.Err != nil {return nil, lexerErrorListener.Err@@ -41,10 +74,11 @@ func ParseCosmosDBQuery(statement string) ([]*ParseResult, error) {return nil, parserErrorListener.Err}- return []*ParseResult{- {- Tree: root,- Tokens: stream,- },- }, nil+ result := &ParseResult{+ Tree: tree,+ Tokens: stream,+ BaseLine: baseLine,+ }++ return result, nil}diff --git a/backend/plugin/parser/cosmosdb/cosmosdb_test.go b/backend/plugin/parser/cosmosdb/cosmosdb_test.gonew file mode 100644index 0000000000..67168d0791--- /dev/null+++ b/backend/plugin/parser/cosmosdb/cosmosdb_test.go@@ -0,0 +1,90 @@+package cosmosdb++import (+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestParseCosmosDBQuery(t *testing.T) {+ tests := []struct {+ name string+ statement string+ wantStatCount int+ wantErr bool+ }{+ {+ name: "Single SELECT statement",+ statement: "SELECT * FROM c",+ wantStatCount: 1,+ wantErr: false,+ },+ {+ name: "SELECT with WHERE clause",+ statement: "SELECT c.id, c.name FROM users c WHERE c.age > 18",+ wantStatCount: 1,+ wantErr: false,+ },+ {+ name: "SELECT with DISTINCT",+ statement: "SELECT DISTINCT c.category FROM products c",+ wantStatCount: 1,+ wantErr: false,+ },+ {+ name: "Empty statement",+ statement: "",+ wantStatCount: 0,+ wantErr: false,+ },+ {+ name: "Only whitespace",+ statement: " \n \t ",+ wantStatCount: 0,+ wantErr: false,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ results, err := ParseCosmosDBQuery(tt.statement)+ if tt.wantErr {+ require.Error(t, err)+ return+ }+ require.NoError(t, err)+ assert.Equal(t, tt.wantStatCount, len(results))++ // Verify each result has the required fields+ for i, result := range results {+ assert.NotNil(t, result.Tree, "Result %d should have a Tree", i)+ assert.NotNil(t, result.Tokens, "Result %d should have Tokens", i)+ assert.Equal(t, 0, result.BaseLine, "Result %d should have BaseLine 0", i)+ }+ })+ }+}++func TestParseCosmosDBQueryErrors(t *testing.T) {+ tests := []struct {+ name string+ statement string+ }{+ {+ name: "Invalid SQL syntax",+ statement: "SELCT * FRM c",+ },+ {+ name: "Unclosed string",+ statement: "SELECT * FROM c WHERE c.name = 'test",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ _, err := ParseCosmosDBQuery(tt.statement)+ require.Error(t, err, "Expected error for invalid CosmosDB SQL")+ })+ }+}diff --git a/backend/plugin/parser/cosmosdb/query_span.go b/backend/plugin/parser/cosmosdb/query_span.goindex b6aef23943..0005cc0cb5 100644--- a/backend/plugin/parser/cosmosdb/query_span.go+++ b/backend/plugin/parser/cosmosdb/query_span.go@@ -28,20 +28,27 @@ func getQuerySpanImpl(statement string) (*base.QuerySpan, error) {}if len(parseResults) == 0 {- return nil, errors.New("no parse result")+ return &base.QuerySpan{+ Results: []base.QuerySpanResult{},+ PredicatePaths: nil,+ }, nil+ }++ if len(parseResults) != 1 {+ return nil, errors.Errorf("expecting only one statement to get query span, but got %d", len(parseResults))}ast := parseResults[0].TreequerySpanResultListener := &querySpanResultListener{}antlr.ParseTreeWalkerDefault.Walk(querySpanResultListener, ast)if querySpanResultListener.err != nil {- return nil, err+ return nil, querySpanResultListener.err}querySpanPredicatePathsListener := &querySpanPredicatePathsListener{}antlr.ParseTreeWalkerDefault.Walk(querySpanPredicatePathsListener, ast)if querySpanPredicatePathsListener.err != nil {- return nil, err+ return nil, querySpanPredicatePathsListener.err}return &base.QuerySpan{Results: querySpanResultListener.result,diff --git a/backend/plugin/parser/cosmosdb/split.go b/backend/plugin/parser/cosmosdb/split.goindex d94379db8d..7e65122f71 100644--- a/backend/plugin/parser/cosmosdb/split.go+++ b/backend/plugin/parser/cosmosdb/split.go@@ -1,6 +1,12 @@package cosmosdbimport (+ "strings"++ "github.com/antlr4-go/antlr/v4"+ parser "github.com/bytebase/parser/cosmosdb"++ "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -10,11 +16,70 @@ func init() {}// SplitSQL splits the given SQL statement into multiple SQL statements.+// Note: CosmosDB only supports single SELECT statements, so this returns the entire input as one statement.func SplitSQL(statement string) ([]base.SingleSQL, error) {- // TODO(zp): Does CosmosDB support multiple statements? And how we use the split function?+ // CosmosDB doesn't support multiple statements or semicolon delimiters.+ // The grammar only accepts: root: select EOF+ if strings.TrimSpace(statement) == "" {+ return nil, nil+ }++ // Use lexer to get proper position information+ lexer := parser.NewCosmosDBLexer(antlr.NewInputStream(statement))+ stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)+ stream.Fill()++ tokens := stream.GetAllTokens()+ if len(tokens) == 0 {+ return nil, nil+ }++ // Check if statement is empty (only whitespace/EOF)+ empty := true+ for _, token := range tokens {+ if token.GetTokenType() == antlr.TokenEOF {+ continue+ }+ if token.GetChannel() != antlr.TokenHiddenChannel {+ empty = false+ break+ }+ }++ var firstToken, lastToken antlr.Token+ for _, token := range tokens {+ if token.GetTokenType() == antlr.TokenEOF {+ lastToken = token+ break+ }+ if firstToken == nil && token.GetChannel() == antlr.TokenDefaultChannel {+ firstToken = token+ }+ if token.GetChannel() == antlr.TokenDefaultChannel {+ lastToken = token+ }+ }++ if firstToken == nil {+ firstToken = tokens[0]+ }+ if lastToken == nil && len(tokens) > 0 {+ lastToken = tokens[len(tokens)-1]+ }+return []base.SingleSQL{{- Text: statement,+ Text: statement,+ BaseLine: 0,+ Start: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{+ Line: int32(firstToken.GetLine()),+ Column: int32(firstToken.GetColumn()),+ }, statement),+ End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{+ Line: int32(lastToken.GetLine()),+ Column: int32(lastToken.GetColumn()),+ }, statement),+ Empty: empty,},}, nil}diff --git a/backend/plugin/parser/cosmosdb/split_test.go b/backend/plugin/parser/cosmosdb/split_test.gonew file mode 100644index 0000000000..0d748a2fd8--- /dev/null+++ b/backend/plugin/parser/cosmosdb/split_test.go@@ -0,0 +1,55 @@+package cosmosdb++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestSplitSQL(t *testing.T) {+ tests := []struct {+ name string+ statement string+ wantCount int+ wantEmpty bool+ }{+ {+ name: "Single SELECT statement",+ statement: "SELECT * FROM c",+ wantCount: 1,+ wantEmpty: false,+ },+ {+ name: "SELECT with WHERE",+ statement: "SELECT c.id FROM users c WHERE c.active = true",+ wantCount: 1,+ wantEmpty: false,+ },+ {+ name: "Empty string",+ statement: "",+ wantCount: 0,+ },+ {+ name: "Only whitespace",+ statement: " \n \t ",+ wantCount: 0,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ list, err := SplitSQL(tt.statement)+ require.NoError(t, err)+ require.Equal(t, tt.wantCount, len(list))++ if tt.wantCount > 0 {+ require.Equal(t, tt.statement, list[0].Text)+ require.Equal(t, 0, list[0].BaseLine)+ require.Equal(t, tt.wantEmpty, list[0].Empty)+ require.NotNil(t, list[0].Start)+ require.NotNil(t, list[0].End)+ }+ })+ }+}diff --git a/go.mod b/go.modindex 4d53a97c2d..325281154e 100644--- a/go.mod+++ b/go.mod@@ -293,7 +293,7 @@ require (github.com/aws/aws-sdk-go-v2/service/sts v1.39.0github.com/aws/smithy-go v1.23.1 // indirectgithub.com/beorn7/perks v1.0.1 // indirect- github.com/bytebase/parser v0.0.0-20251113094200-734ef85da6ad+ github.com/bytebase/parser v0.0.0-20251117032631-393e2917be3fgithub.com/cespare/xxhash/v2 v2.3.0 // indirectgithub.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirectgithub.com/cockroachdb/cockroach-go/v2 v2.4.2diff --git a/go.sum b/go.sumindex bee2d0f8a1..6abef793db 100644--- a/go.sum+++ b/go.sum@@ -853,8 +853,8 @@ github.com/bytebase/gosasl v0.0.0-20240422091407-6b7481e86f08 h1:S1GgjJLspz4E4z4github.com/bytebase/gosasl v0.0.0-20240422091407-6b7481e86f08/go.mod h1:Qx8cW6jkI8riyzmklj80kAIkv+iezFUTBiGU0qHhHes=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/parser v0.0.0-20251113094200-734ef85da6ad h1:cemWTRG6+M3LPA4h4Dl0wJIG90KX4OmV+BFrSDvCiDk=-github.com/bytebase/parser v0.0.0-20251113094200-734ef85da6ad/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=+github.com/bytebase/parser v0.0.0-20251117032631-393e2917be3f h1:1Zo9bMTS/VMpUC4IcNpqUjj+rzyAJ1ktWjcSOALYRlU=+github.com/bytebase/parser v0.0.0-20251117032631-393e2917be3f/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=github.com/bytebase/pkcs8 v0.0.0-20240612095628-fcd0a7484c94 h1:6PNsuNmSqCuR8Z616uQLzDIvujWCFsKzeVc3ECmVubk=