Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: unify document DB predicate paths in QuerySpan (#19585)

h3n4l17b38093bb0b
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index 0f5e827493..1c1410d5eb 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -592,7 +592,7 @@ func queryRetry(
// Pre-execution check for MongoDB: enforce Milestone 1 supported APIs and reject
// sensitive predicates before sending the query to the server.
if maskingEnabled && instance.Metadata.GetEngine() == storepb.Engine_MONGODB {
- for _, stmt := range statements {
+ for i, stmt := range statements {
analysis, analyzeErr := mongoparser.AnalyzeMaskingStatement(stmt.Text)
if analyzeErr != nil || analysis == nil || analysis.Collection == "" {
continue
@@ -610,11 +610,13 @@ func queryRetry(
return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)
}
- for _, field := range analysis.PredicateFields {
- semanticType := lookupSemanticTypeByDotPath(field, objectSchema)
- if semanticType != "" {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,
- errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", field, semanticType))
+ if i < len(spans) {
+ for pathStr := range spans[i].PredicatePaths {
+ semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)
+ if semanticType != "" {
+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,
+ errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType))
+ }
}
}
}
@@ -623,7 +625,7 @@ func queryRetry(
// Pre-execution check for Elasticsearch: reject blocked APIs and predicate violations
// before sending the query to the server.
if maskingEnabled && instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH {
- for _, stmt := range statements {
+ for i, stmt := range statements {
parsed, parseErr := esparser.ParseElasticsearchREST(stmt.Text)
if parseErr != nil || len(parsed.Requests) == 0 {
continue
@@ -647,11 +649,13 @@ func queryRetry(
if err := checkElasticsearchRequestBlocked(analysis); err != nil {
return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)
}
- for _, field := range analysis.PredicateFields {
- semanticType := lookupSemanticTypeByDotPath(field, objectSchema)
- if semanticType != "" {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,
- errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", field, semanticType))
+ if i < len(spans) {
+ for pathStr := range spans[i].PredicatePaths {
+ semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)
+ if semanticType != "" {
+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,
+ errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType))
+ }
}
}
}
diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.go
index 52d9b280f7..13e2377209 100644
--- a/backend/plugin/parser/base/span.go
+++ b/backend/plugin/parser/base/span.go
@@ -63,8 +63,9 @@ type QuerySpan struct {
// PredicateColumns are the source columns contributing to the span.
// PredicateColumns here are the source columns for the where conditions.
PredicateColumns SourceColumnSet
- // PredicatePaths is predicateColumns used by Cosmos DB only, to store the path of the field, all the
- // paths should begin with the container name.
+ // PredicatePaths stores the paths of fields used in query predicates for document databases
+ // (CosmosDB, MongoDB, Elasticsearch). For CosmosDB, paths begin with the container name.
+ // For MongoDB and Elasticsearch, paths are dot-delimited field names (e.g. "contact.phone").
PredicatePaths map[string]*PathAST
NotFoundError error
FunctionNotSupportedError error
@@ -412,6 +413,10 @@ func (s *QuerySpan) ToYaml() *YamlQuerySpan {
}
return 0
})
+ for k := range s.PredicatePaths {
+ y.PredicatePaths = append(y.PredicatePaths, k)
+ }
+ slices.Sort(y.PredicatePaths)
return y
}
@@ -420,6 +425,7 @@ type YamlQuerySpan struct {
Results []YamlQuerySpanResult
SourceColumns []ColumnResource
PredicateColumns []ColumnResource
+ PredicatePaths []string
}
type YamlQuerySpanResult struct {
diff --git a/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml b/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml
index 45e3ff12de..27e6cce907 100644
--- a/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml
+++ b/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml
@@ -14,3 +14,4 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
+ predicatepaths: []
diff --git a/backend/plugin/parser/elasticsearch/query_span.go b/backend/plugin/parser/elasticsearch/query_span.go
index d36cabc126..35b1bd48af 100644
--- a/backend/plugin/parser/elasticsearch/query_span.go
+++ b/backend/plugin/parser/elasticsearch/query_span.go
@@ -2,6 +2,7 @@ package elasticsearch
import (
"context"
+ "strings"
"github.com/pkg/errors"
@@ -44,5 +45,32 @@ func GetQuerySpan(
req := parseResult.Requests[0]
queryType := ClassifyRequest(req.Method, req.URL)
- return &base.QuerySpan{Type: queryType}, nil
+ span := &base.QuerySpan{Type: queryType}
+
+ analysis := AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))
+ if len(analysis.PredicateFields) > 0 {
+ span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))
+ for _, field := range analysis.PredicateFields {
+ span.PredicatePaths[field] = dotPathToPathAST(field)
+ }
+ }
+
+ return span, nil
+}
+
+// dotPathToPathAST converts a dot-delimited field path (e.g. "contact.phone")
+// into a PathAST with ItemSelector nodes.
+func dotPathToPathAST(dotPath string) *base.PathAST {
+ parts := strings.Split(dotPath, ".")
+ if len(parts) == 0 {
+ return nil
+ }
+ root := base.NewItemSelector(parts[0])
+ current := base.SelectorNode(root)
+ for _, part := range parts[1:] {
+ next := base.NewItemSelector(part)
+ current.SetNext(next)
+ current = next
+ }
+ return base.NewPathAST(root)
}
diff --git a/backend/plugin/parser/elasticsearch/query_span_test.go b/backend/plugin/parser/elasticsearch/query_span_test.go
index a7aa4b5302..c81a8587c9 100644
--- a/backend/plugin/parser/elasticsearch/query_span_test.go
+++ b/backend/plugin/parser/elasticsearch/query_span_test.go
@@ -57,6 +57,46 @@ func TestGetQuerySpan(t *testing.T) {
}
}
+func TestGetQuerySpanPredicatePaths(t *testing.T) {
+ tests := []struct {
+ description string
+ statement string
+ predicatePaths map[string]bool
+ }{
+ {
+ description: "search with match predicate",
+ statement: "GET /users/_search\n{\"query\":{\"match\":{\"email\":\"alice\"}}}",
+ predicatePaths: map[string]bool{
+ "email": true,
+ },
+ },
+ {
+ description: "search with nested bool query",
+ statement: "GET /users/_search\n{\"query\":{\"bool\":{\"must\":[{\"term\":{\"status\":\"active\"}},{\"range\":{\"age\":{\"gte\":18}}}]}}}",
+ predicatePaths: map[string]bool{
+ "status": true,
+ "age": true,
+ },
+ },
+ {
+ description: "get doc returns no predicates",
+ statement: "GET /users/_doc/123",
+ predicatePaths: map[string]bool{},
+ },
+ }
+
+ a := require.New(t)
+ for _, tc := range tests {
+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{}, base.Statement{Text: tc.statement}, "", "", false)
+ a.NoError(err, tc.description)
+ a.Equal(len(tc.predicatePaths), len(span.PredicatePaths), tc.description)
+ for path := range tc.predicatePaths {
+ _, ok := span.PredicatePaths[path]
+ a.True(ok, "%s: missing path %q", tc.description, path)
+ }
+ }
+}
+
func TestGetQuerySpan_Error(t *testing.T) {
// MongoDB style query, definitely not ElasticSearch
stmt := "db.users.find({})"
diff --git a/backend/plugin/parser/mongodb/query_span.go b/backend/plugin/parser/mongodb/query_span.go
index 00da8cee6e..9c550f24e6 100644
--- a/backend/plugin/parser/mongodb/query_span.go
+++ b/backend/plugin/parser/mongodb/query_span.go
@@ -2,6 +2,7 @@ package mongodb
import (
"context"
+ "strings"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -12,11 +13,45 @@ func init() {
}
// GetQuerySpan returns a QuerySpan for a MongoDB shell statement.
-// It classifies the statement type for access control purposes.
-// MongoDB does not support column-level tracking, so only the QueryType is set.
+// It classifies the statement type and populates PredicatePaths from
+// predicate fields extracted by masking analysis.
func GetQuerySpan(_ context.Context, _ base.GetQuerySpanContext, stmt base.Statement, _ string, _ string, _ bool) (*base.QuerySpan, error) {
queryType := GetQueryType(stmt.Text)
- return &base.QuerySpan{
+ span := &base.QuerySpan{
Type: queryType,
- }, nil
+ }
+
+ analysis, err := AnalyzeMaskingStatement(stmt.Text)
+ if err != nil {
+ return span, err
+ }
+ if analysis == nil {
+ return span, nil
+ }
+
+ if len(analysis.PredicateFields) > 0 {
+ span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))
+ for _, field := range analysis.PredicateFields {
+ span.PredicatePaths[field] = dotPathToPathAST(field)
+ }
+ }
+
+ return span, nil
+}
+
+// dotPathToPathAST converts a dot-delimited field path (e.g. "contact.phone")
+// into a PathAST with ItemSelector nodes.
+func dotPathToPathAST(dotPath string) *base.PathAST {
+ parts := strings.Split(dotPath, ".")
+ if len(parts) == 0 {
+ return nil
+ }
+ root := base.NewItemSelector(parts[0])
+ current := base.SelectorNode(root)
+ for _, part := range parts[1:] {
+ next := base.NewItemSelector(part)
+ current.SetNext(next)
+ current = next
+ }
+ return base.NewPathAST(root)
}
diff --git a/backend/plugin/parser/mongodb/query_span_test.go b/backend/plugin/parser/mongodb/query_span_test.go
index 55222e0b82..40e061581b 100644
--- a/backend/plugin/parser/mongodb/query_span_test.go
+++ b/backend/plugin/parser/mongodb/query_span_test.go
@@ -9,6 +9,57 @@ import (
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
+func TestGetQuerySpanPredicatePaths(t *testing.T) {
+ tests := []struct {
+ description string
+ statement string
+ predicatePaths map[string]bool
+ }{
+ {
+ description: "find with single predicate field",
+ statement: `db.users.find({email: "[email protected]"})`,
+ predicatePaths: map[string]bool{
+ "email": true,
+ },
+ },
+ {
+ description: "findOne with nested dot path",
+ statement: `db["users"].findOne({"contact.phone": "123"})`,
+ predicatePaths: map[string]bool{
+ "contact.phone": true,
+ },
+ },
+ {
+ description: "aggregate with $match predicate",
+ statement: `db.users.aggregate([{$match: {name: "alice"}}])`,
+ predicatePaths: map[string]bool{
+ "name": true,
+ },
+ },
+ {
+ description: "find with no predicate",
+ statement: `db.users.find()`,
+ predicatePaths: map[string]bool{},
+ },
+ {
+ description: "non-read operation returns no paths",
+ statement: `db.users.insertOne({name: "bob"})`,
+ predicatePaths: map[string]bool{},
+ },
+ }
+
+ a := require.New(t)
+ for _, tc := range tests {
+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{}, base.Statement{Text: tc.statement}, "", "", false)
+ a.NoError(err, tc.description)
+ a.Equal(len(tc.predicatePaths), len(span.PredicatePaths), tc.description)
+ for path := range tc.predicatePaths {
+ _, ok := span.PredicatePaths[path]
+ a.True(ok, "%s: missing path %q", tc.description, path)
+ }
+ }
+}
+
func TestGetQuerySpan(t *testing.T) {
tests := []struct {
description string