Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: unify document DB predicate paths in QuerySpan (#19585)
h3n4l17b38093bb0b
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex 0f5e827493..1c1410d5eb 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -592,7 +592,7 @@ func queryRetry(// Pre-execution check for MongoDB: enforce Milestone 1 supported APIs and reject// sensitive predicates before sending the query to the server.if maskingEnabled && instance.Metadata.GetEngine() == storepb.Engine_MONGODB {- for _, stmt := range statements {+ for i, stmt := range statements {analysis, analyzeErr := mongoparser.AnalyzeMaskingStatement(stmt.Text)if analyzeErr != nil || analysis == nil || analysis.Collection == "" {continue@@ -610,11 +610,13 @@ func queryRetry(return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)}- for _, field := range analysis.PredicateFields {- semanticType := lookupSemanticTypeByDotPath(field, objectSchema)- if semanticType != "" {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,- errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", field, semanticType))+ if i < len(spans) {+ for pathStr := range spans[i].PredicatePaths {+ semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)+ if semanticType != "" {+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,+ errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType))+ }}}}@@ -623,7 +625,7 @@ func queryRetry(// Pre-execution check for Elasticsearch: reject blocked APIs and predicate violations// before sending the query to the server.if maskingEnabled && instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH {- for _, stmt := range statements {+ for i, stmt := range statements {parsed, parseErr := esparser.ParseElasticsearchREST(stmt.Text)if parseErr != nil || len(parsed.Requests) == 0 {continue@@ -647,11 +649,13 @@ func queryRetry(if err := checkElasticsearchRequestBlocked(analysis); err != nil {return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)}- for _, field := range analysis.PredicateFields {- semanticType := lookupSemanticTypeByDotPath(field, objectSchema)- if semanticType != "" {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,- errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", field, semanticType))+ if i < len(spans) {+ for pathStr := range spans[i].PredicatePaths {+ semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)+ if semanticType != "" {+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,+ errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType))+ }}}}diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.goindex 52d9b280f7..13e2377209 100644--- a/backend/plugin/parser/base/span.go+++ b/backend/plugin/parser/base/span.go@@ -63,8 +63,9 @@ type QuerySpan struct {// PredicateColumns are the source columns contributing to the span.// PredicateColumns here are the source columns for the where conditions.PredicateColumns SourceColumnSet- // PredicatePaths is predicateColumns used by Cosmos DB only, to store the path of the field, all the- // paths should begin with the container name.+ // PredicatePaths stores the paths of fields used in query predicates for document databases+ // (CosmosDB, MongoDB, Elasticsearch). For CosmosDB, paths begin with the container name.+ // For MongoDB and Elasticsearch, paths are dot-delimited field names (e.g. "contact.phone").PredicatePaths map[string]*PathASTNotFoundError errorFunctionNotSupportedError error@@ -412,6 +413,10 @@ func (s *QuerySpan) ToYaml() *YamlQuerySpan {}return 0})+ for k := range s.PredicatePaths {+ y.PredicatePaths = append(y.PredicatePaths, k)+ }+ slices.Sort(y.PredicatePaths)return y}@@ -420,6 +425,7 @@ type YamlQuerySpan struct {Results []YamlQuerySpanResultSourceColumns []ColumnResourcePredicateColumns []ColumnResource+ PredicatePaths []string}type YamlQuerySpanResult struct {diff --git a/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml b/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yamlindex 45e3ff12de..27e6cce907 100644--- a/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml+++ b/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml@@ -14,3 +14,4 @@selectasterisk: falsesourcecolumns: []predicatecolumns: []+ predicatepaths: []diff --git a/backend/plugin/parser/elasticsearch/query_span.go b/backend/plugin/parser/elasticsearch/query_span.goindex d36cabc126..35b1bd48af 100644--- a/backend/plugin/parser/elasticsearch/query_span.go+++ b/backend/plugin/parser/elasticsearch/query_span.go@@ -2,6 +2,7 @@ package elasticsearchimport ("context"+ "strings""github.com/pkg/errors"@@ -44,5 +45,32 @@ func GetQuerySpan(req := parseResult.Requests[0]queryType := ClassifyRequest(req.Method, req.URL)- return &base.QuerySpan{Type: queryType}, nil+ span := &base.QuerySpan{Type: queryType}++ analysis := AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))+ if len(analysis.PredicateFields) > 0 {+ span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))+ for _, field := range analysis.PredicateFields {+ span.PredicatePaths[field] = dotPathToPathAST(field)+ }+ }++ return span, nil+}++// dotPathToPathAST converts a dot-delimited field path (e.g. "contact.phone")+// into a PathAST with ItemSelector nodes.+func dotPathToPathAST(dotPath string) *base.PathAST {+ parts := strings.Split(dotPath, ".")+ if len(parts) == 0 {+ return nil+ }+ root := base.NewItemSelector(parts[0])+ current := base.SelectorNode(root)+ for _, part := range parts[1:] {+ next := base.NewItemSelector(part)+ current.SetNext(next)+ current = next+ }+ return base.NewPathAST(root)}diff --git a/backend/plugin/parser/elasticsearch/query_span_test.go b/backend/plugin/parser/elasticsearch/query_span_test.goindex a7aa4b5302..c81a8587c9 100644--- a/backend/plugin/parser/elasticsearch/query_span_test.go+++ b/backend/plugin/parser/elasticsearch/query_span_test.go@@ -57,6 +57,46 @@ func TestGetQuerySpan(t *testing.T) {}}+func TestGetQuerySpanPredicatePaths(t *testing.T) {+ tests := []struct {+ description string+ statement string+ predicatePaths map[string]bool+ }{+ {+ description: "search with match predicate",+ statement: "GET /users/_search\n{\"query\":{\"match\":{\"email\":\"alice\"}}}",+ predicatePaths: map[string]bool{+ "email": true,+ },+ },+ {+ description: "search with nested bool query",+ statement: "GET /users/_search\n{\"query\":{\"bool\":{\"must\":[{\"term\":{\"status\":\"active\"}},{\"range\":{\"age\":{\"gte\":18}}}]}}}",+ predicatePaths: map[string]bool{+ "status": true,+ "age": true,+ },+ },+ {+ description: "get doc returns no predicates",+ statement: "GET /users/_doc/123",+ predicatePaths: map[string]bool{},+ },+ }++ a := require.New(t)+ for _, tc := range tests {+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{}, base.Statement{Text: tc.statement}, "", "", false)+ a.NoError(err, tc.description)+ a.Equal(len(tc.predicatePaths), len(span.PredicatePaths), tc.description)+ for path := range tc.predicatePaths {+ _, ok := span.PredicatePaths[path]+ a.True(ok, "%s: missing path %q", tc.description, path)+ }+ }+}+func TestGetQuerySpan_Error(t *testing.T) {// MongoDB style query, definitely not ElasticSearchstmt := "db.users.find({})"diff --git a/backend/plugin/parser/mongodb/query_span.go b/backend/plugin/parser/mongodb/query_span.goindex 00da8cee6e..9c550f24e6 100644--- a/backend/plugin/parser/mongodb/query_span.go+++ b/backend/plugin/parser/mongodb/query_span.go@@ -2,6 +2,7 @@ package mongodbimport ("context"+ "strings"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -12,11 +13,45 @@ func init() {}// GetQuerySpan returns a QuerySpan for a MongoDB shell statement.-// It classifies the statement type for access control purposes.-// MongoDB does not support column-level tracking, so only the QueryType is set.+// It classifies the statement type and populates PredicatePaths from+// predicate fields extracted by masking analysis.func GetQuerySpan(_ context.Context, _ base.GetQuerySpanContext, stmt base.Statement, _ string, _ string, _ bool) (*base.QuerySpan, error) {queryType := GetQueryType(stmt.Text)- return &base.QuerySpan{+ span := &base.QuerySpan{Type: queryType,- }, nil+ }++ analysis, err := AnalyzeMaskingStatement(stmt.Text)+ if err != nil {+ return span, err+ }+ if analysis == nil {+ return span, nil+ }++ if len(analysis.PredicateFields) > 0 {+ span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))+ for _, field := range analysis.PredicateFields {+ span.PredicatePaths[field] = dotPathToPathAST(field)+ }+ }++ return span, nil+}++// dotPathToPathAST converts a dot-delimited field path (e.g. "contact.phone")+// into a PathAST with ItemSelector nodes.+func dotPathToPathAST(dotPath string) *base.PathAST {+ parts := strings.Split(dotPath, ".")+ if len(parts) == 0 {+ return nil+ }+ root := base.NewItemSelector(parts[0])+ current := base.SelectorNode(root)+ for _, part := range parts[1:] {+ next := base.NewItemSelector(part)+ current.SetNext(next)+ current = next+ }+ return base.NewPathAST(root)}diff --git a/backend/plugin/parser/mongodb/query_span_test.go b/backend/plugin/parser/mongodb/query_span_test.goindex 55222e0b82..40e061581b 100644--- a/backend/plugin/parser/mongodb/query_span_test.go+++ b/backend/plugin/parser/mongodb/query_span_test.go@@ -9,6 +9,57 @@ import ("github.com/bytebase/bytebase/backend/plugin/parser/base")+func TestGetQuerySpanPredicatePaths(t *testing.T) {+ tests := []struct {+ description string+ statement string+ predicatePaths map[string]bool+ }{+ {+ description: "find with single predicate field",+ statement: `db.users.find({email: "[email protected]"})`,+ predicatePaths: map[string]bool{+ "email": true,+ },+ },+ {+ description: "findOne with nested dot path",+ statement: `db["users"].findOne({"contact.phone": "123"})`,+ predicatePaths: map[string]bool{+ "contact.phone": true,+ },+ },+ {+ description: "aggregate with $match predicate",+ statement: `db.users.aggregate([{$match: {name: "alice"}}])`,+ predicatePaths: map[string]bool{+ "name": true,+ },+ },+ {+ description: "find with no predicate",+ statement: `db.users.find()`,+ predicatePaths: map[string]bool{},+ },+ {+ description: "non-read operation returns no paths",+ statement: `db.users.insertOne({name: "bob"})`,+ predicatePaths: map[string]bool{},+ },+ }++ a := require.New(t)+ for _, tc := range tests {+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{}, base.Statement{Text: tc.statement}, "", "", false)+ a.NoError(err, tc.description)+ a.Equal(len(tc.predicatePaths), len(span.PredicatePaths), tc.description)+ for path := range tc.predicatePaths {+ _, ok := span.PredicatePaths[path]+ a.True(ok, "%s: missing path %q", tc.description, path)+ }+ }+}+func TestGetQuerySpan(t *testing.T) {tests := []struct {description string