Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: cache document DB analysis on QuerySpan and unify predicate checks (#19590)

h3n4l3f96d9002e88
diff --git a/backend/api/v1/catalog_masking.go b/backend/api/v1/catalog_masking.go
index 757bff7ef7..a31d55afdc 100644
--- a/backend/api/v1/catalog_masking.go
+++ b/backend/api/v1/catalog_masking.go
@@ -1,23 +1,151 @@
package v1
import (
+ "context"
"encoding/json"
+ "fmt"
+ "connectrpc.com/connect"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/component/masker"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
- "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store"
)
-func getFirstSemanticTypeInPath(ast *base.PathAST, objectSchema *storepb.ObjectSchema) string {
+// preExecuteMaskingCheck runs engine-specific pre-execution checks that block
+// queries with unsupported APIs. Sensitive predicate checks are handled
+// post-execution on a per-statement basis by checkSensitivePredicates.
+func preExecuteMaskingCheck(
+ ctx context.Context,
+ stores *store.Store,
+ engine storepb.Engine,
+ database *store.DatabaseMessage,
+ spans []*parserbase.QuerySpan,
+) error {
+ switch engine {
+ case storepb.Engine_MONGODB:
+ return preExecuteMaskingCheckMongoDB(ctx, stores, database, spans)
+ case storepb.Engine_ELASTICSEARCH:
+ return preExecuteMaskingCheckElasticsearch(ctx, stores, database, spans)
+ default:
+ return nil
+ }
+}
+
+func preExecuteMaskingCheckMongoDB(
+ ctx context.Context,
+ stores *store.Store,
+ database *store.DatabaseMessage,
+ spans []*parserbase.QuerySpan,
+) error {
+ for _, span := range spans {
+ if span.MongoDBAnalysis == nil {
+ continue
+ }
+ analysis := span.MongoDBAnalysis
+ if analysis.Collection == "" {
+ continue
+ }
+
+ objectSchema, err := getMongoDBCollectionObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, analysis.Collection)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get object schema for collection %q", analysis.Collection))
+ }
+ if objectSchema == nil {
+ continue
+ }
+
+ if err := checkMongoDBRequestBlocked(analysis); err != nil {
+ return connect.NewError(connect.CodeInvalidArgument, err)
+ }
+ }
+ return nil
+}
+
+func preExecuteMaskingCheckElasticsearch(
+ ctx context.Context,
+ stores *store.Store,
+ database *store.DatabaseMessage,
+ spans []*parserbase.QuerySpan,
+) error {
+ for _, span := range spans {
+ if span.ElasticsearchAnalysis == nil {
+ continue
+ }
+ analysis := span.ElasticsearchAnalysis
+ if analysis.API == parserbase.ElasticsearchAPIUnsupported {
+ continue
+ }
+ indexName := analysis.Index
+ if indexName == "" {
+ continue
+ }
+ objectSchema, err := getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, indexName)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get object schema for index %q", indexName))
+ }
+ if objectSchema == nil || !objectSchemaHasSemanticTypes(objectSchema) {
+ continue
+ }
+ if err := checkElasticsearchRequestBlocked(analysis); err != nil {
+ return connect.NewError(connect.CodeInvalidArgument, err)
+ }
+ }
+ return nil
+}
+
+// checkSensitivePredicates checks whether a span uses sensitive fields in
+// predicates. Returns an error message if so, or "" if clean. This is used
+// post-execution to blank individual statement results (matching SQL/CosmosDB
+// behavior) rather than blocking the entire request.
+func checkSensitivePredicates(
+ ctx context.Context,
+ stores *store.Store,
+ database *store.DatabaseMessage,
+ span *parserbase.QuerySpan,
+) (string, error) {
+ if len(span.PredicatePaths) == 0 {
+ return "", nil
+ }
+
+ var objectSchema *storepb.ObjectSchema
+ var err error
+
+ if analysis := span.MongoDBAnalysis; analysis != nil && analysis.Collection != "" {
+ objectSchema, err = getMongoDBCollectionObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, analysis.Collection)
+ if err != nil {
+ return "", errors.Wrapf(err, "failed to get object schema for collection %q", analysis.Collection)
+ }
+ } else if analysis := span.ElasticsearchAnalysis; analysis != nil && analysis.Index != "" {
+ objectSchema, err = getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, analysis.Index)
+ if err != nil {
+ return "", errors.Wrapf(err, "failed to get object schema for index %q", analysis.Index)
+ }
+ }
+
+ if objectSchema == nil {
+ return "", nil
+ }
+
+ for pathStr := range span.PredicatePaths {
+ semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)
+ if semanticType != "" {
+ return fmt.Sprintf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType), nil
+ }
+ }
+ return "", nil
+}
+
+func getFirstSemanticTypeInPath(ast *parserbase.PathAST, objectSchema *storepb.ObjectSchema) string {
if ast == nil || ast.Root == nil || objectSchema == nil {
return ""
}
// Skip the first node because it always represents the container.
- astWoutContainer := base.NewPathAST(ast.Root.GetNext())
+ astWoutContainer := parserbase.NewPathAST(ast.Root.GetNext())
if astWoutContainer == nil || astWoutContainer.Root == nil {
return ""
}
@@ -34,7 +162,7 @@ func getFirstSemanticTypeInPath(ast *base.PathAST, objectSchema *storepb.ObjectS
}
switch node := node.(type) {
- case *base.ItemSelector:
+ case *parserbase.ItemSelector:
if os.Type != storepb.ObjectSchema_OBJECT {
return ""
}
@@ -48,7 +176,7 @@ func getFirstSemanticTypeInPath(ast *base.PathAST, objectSchema *storepb.ObjectS
if !valid {
return ""
}
- case *base.ArraySelector:
+ case *parserbase.ArraySelector:
if os.Type != storepb.ObjectSchema_OBJECT {
return ""
}
@@ -82,30 +210,30 @@ func getFirstSemanticTypeInPath(ast *base.PathAST, objectSchema *storepb.ObjectS
return ""
}
-func maskCosmosDB(span *base.QuerySpan, data map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {
+func maskCosmosDB(span *parserbase.QuerySpan, data map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {
if len(span.Results) != 1 {
return nil, errors.Errorf("expected 1 result, but got %d", len(span.Results))
}
return walkAndMaskJSON(data, span.Results[0].SourceFieldPaths, objectSchema, semanticTypeToMasker)
}
-func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*base.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {
+func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*parserbase.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {
result := make(map[string]any)
for key, value := range data {
o := objectSchema
- var ast *base.PathAST
+ var ast *parserbase.PathAST
if fieldPaths != nil {
// Relocate the object schema cursor to the path position.
// Skip the first node because it always represents the container.
if path, ok := fieldPaths[key]; ok {
if path != nil && path.Root != nil {
- astWoutContainer := base.NewPathAST(path.Root.GetNext())
+ astWoutContainer := parserbase.NewPathAST(path.Root.GetNext())
ast = astWoutContainer
}
}
}
if ast == nil || ast.Root == nil {
- ast = base.NewPathAST(base.NewItemSelector(key))
+ ast = parserbase.NewPathAST(parserbase.NewItemSelector(key))
}
var parentSemanticType string
@@ -130,7 +258,7 @@ func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*base.PathAST, o
return result, nil
}
-func getObjectSchemaByPath(objectSchema *storepb.ObjectSchema, path *base.PathAST) (*storepb.ObjectSchema, string) {
+func getObjectSchemaByPath(objectSchema *storepb.ObjectSchema, path *parserbase.PathAST) (*storepb.ObjectSchema, string) {
outer := objectSchema
outerSemanticType := outer.SemanticType
if outerSemanticType != "" {
diff --git a/backend/api/v1/catalog_masking_elasticsearch.go b/backend/api/v1/catalog_masking_elasticsearch.go
index bbc49f56d4..d35e1ce817 100644
--- a/backend/api/v1/catalog_masking_elasticsearch.go
+++ b/backend/api/v1/catalog_masking_elasticsearch.go
@@ -9,7 +9,7 @@ import (
"github.com/bytebase/bytebase/backend/component/masker"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"
+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store"
)
@@ -416,14 +416,14 @@ func maskElasticsearchMSearchSingleResponse(respMap map[string]any, sortFields [
// checkElasticsearchRequestBlocked returns an error if the request must be blocked
// because masking policies exist on the target index.
-func checkElasticsearchRequestBlocked(analysis *esparser.RequestAnalysis) error {
- if analysis.API == esparser.APIBlocked {
+func checkElasticsearchRequestBlocked(analysis *parserbase.ElasticsearchAnalysis) error {
+ if analysis.API == parserbase.ElasticsearchAPIBlocked {
return errors.New("this Elasticsearch API is not supported when data masking is configured on the target index")
}
if len(analysis.BlockedFeatures) > 0 {
names := make([]string, 0, len(analysis.BlockedFeatures))
for _, f := range analysis.BlockedFeatures {
- names = append(names, esparser.BlockedFeatureNames[f])
+ names = append(names, parserbase.ElasticsearchBlockedFeatureNames[f])
}
return errors.Errorf("the following features are not supported when data masking is configured: %s", strings.Join(names, ", "))
}
diff --git a/backend/api/v1/catalog_masking_mongodb.go b/backend/api/v1/catalog_masking_mongodb.go
index 35c6eb2031..74c3a97359 100644
--- a/backend/api/v1/catalog_masking_mongodb.go
+++ b/backend/api/v1/catalog_masking_mongodb.go
@@ -8,14 +8,14 @@ import (
"github.com/bytebase/bytebase/backend/component/masker"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- mongoparser "github.com/bytebase/bytebase/backend/plugin/parser/mongodb"
+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
)
-func checkMongoDBRequestBlocked(analysis *mongoparser.MaskingAnalysis) error {
+func checkMongoDBRequestBlocked(analysis *parserbase.MongoDBAnalysis) error {
if analysis == nil {
return nil
}
- if analysis.API == mongoparser.MaskableAPIUnsupportedRead {
+ if analysis.API == parserbase.MongoDBMaskableAPIUnsupportedRead {
operation := analysis.Operation
if operation == "" {
operation = "unknown"
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index 1c1410d5eb..36bcfb8d00 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -32,8 +32,6 @@ import (
"github.com/bytebase/bytebase/backend/plugin/advisor/code"
"github.com/bytebase/bytebase/backend/plugin/db"
parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
- esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"
- mongoparser "github.com/bytebase/bytebase/backend/plugin/parser/mongodb"
"github.com/bytebase/bytebase/backend/plugin/schema"
"github.com/bytebase/bytebase/backend/runner/schemasync"
"github.com/bytebase/bytebase/backend/store"
@@ -589,75 +587,9 @@ func queryRetry(
maskingEnabled := !queryContext.Explain && !queryContext.SkipMasking &&
licenseService.IsFeatureEnabledForInstance(v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil
- // Pre-execution check for MongoDB: enforce Milestone 1 supported APIs and reject
- // sensitive predicates before sending the query to the server.
- if maskingEnabled && instance.Metadata.GetEngine() == storepb.Engine_MONGODB {
- for i, stmt := range statements {
- analysis, analyzeErr := mongoparser.AnalyzeMaskingStatement(stmt.Text)
- if analyzeErr != nil || analysis == nil || analysis.Collection == "" {
- continue
- }
-
- objectSchema, schemaErr := getMongoDBCollectionObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, analysis.Collection)
- if schemaErr != nil {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.Wrapf(schemaErr, "failed to get object schema for collection %q", analysis.Collection))
- }
- if objectSchema == nil {
- continue
- }
-
- if err := checkMongoDBRequestBlocked(analysis); err != nil {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)
- }
-
- if i < len(spans) {
- for pathStr := range spans[i].PredicatePaths {
- semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)
- if semanticType != "" {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,
- errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType))
- }
- }
- }
- }
- }
-
- // Pre-execution check for Elasticsearch: reject blocked APIs and predicate violations
- // before sending the query to the server.
- if maskingEnabled && instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH {
- for i, stmt := range statements {
- parsed, parseErr := esparser.ParseElasticsearchREST(stmt.Text)
- if parseErr != nil || len(parsed.Requests) == 0 {
- continue
- }
- req := parsed.Requests[0]
- analysis := esparser.AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))
- if analysis.API == esparser.APIUnsupported {
- continue
- }
- indexName := analysis.Index
- if indexName == "" {
- continue
- }
- objectSchema, schemaErr := getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, indexName)
- if schemaErr != nil {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.Wrapf(schemaErr, "failed to get object schema for index %q", indexName))
- }
- if objectSchema == nil || !objectSchemaHasSemanticTypes(objectSchema) {
- continue
- }
- if err := checkElasticsearchRequestBlocked(analysis); err != nil {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)
- }
- if i < len(spans) {
- for pathStr := range spans[i].PredicatePaths {
- semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)
- if semanticType != "" {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,
- errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType))
- }
- }
- }
+ if maskingEnabled {
+ if err := preExecuteMaskingCheck(ctx, stores, instance.Metadata.GetEngine(), database, spans); err != nil {
+ return nil, nil, time.Duration(0), err
}
}
@@ -813,12 +745,26 @@ func queryRetry(
return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.New(err.Error()))
}
- for _, result := range results {
- analysis, analyzeErr := mongoparser.AnalyzeMaskingStatement(result.Statement)
- if analyzeErr != nil || analysis == nil || analysis.Collection == "" {
+ for i, result := range results {
+ if i < len(spans) {
+ if errMsg, err := checkSensitivePredicates(ctx, stores, database, spans[i]); err != nil {
+ return nil, nil, duration, connect.NewError(connect.CodeInternal, err)
+ } else if errMsg != "" {
+ result.Error = errMsg
+ result.Rows = nil
+ result.RowsCount = 0
+ continue
+ }
+ }
+
+ var analysis *parserbase.MongoDBAnalysis
+ if i < len(spans) {
+ analysis = spans[i].MongoDBAnalysis
+ }
+ if analysis == nil || analysis.Collection == "" {
continue
}
- if analysis.API != mongoparser.MaskableAPIFind && analysis.API != mongoparser.MaskableAPIFindOne && analysis.API != mongoparser.MaskableAPIAggregate {
+ if analysis.API != parserbase.MongoDBMaskableAPIFind && analysis.API != parserbase.MongoDBMaskableAPIFindOne && analysis.API != parserbase.MongoDBMaskableAPIAggregate {
continue
}
@@ -864,15 +810,27 @@ func queryRetry(
}
}
} else if instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH {
- for _, result := range results {
- parsed, err := esparser.ParseElasticsearchREST(result.Statement)
- if err != nil || len(parsed.Requests) == 0 {
+ for i, result := range results {
+ if i < len(spans) {
+ if errMsg, err := checkSensitivePredicates(ctx, stores, database, spans[i]); err != nil {
+ return nil, nil, duration, connect.NewError(connect.CodeInternal, err)
+ } else if errMsg != "" {
+ result.Error = errMsg
+ result.Rows = nil
+ result.RowsCount = 0
+ continue
+ }
+ }
+
+ var analysis *parserbase.ElasticsearchAnalysis
+ if i < len(spans) {
+ analysis = spans[i].ElasticsearchAnalysis
+ }
+ if analysis == nil {
continue
}
- req := parsed.Requests[0]
- analysis := esparser.AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))
- if analysis.API == esparser.APIUnsupported || analysis.API == esparser.APIBlocked {
+ if analysis.API == parserbase.ElasticsearchAPIUnsupported || analysis.API == parserbase.ElasticsearchAPIBlocked {
continue
}
@@ -908,7 +866,7 @@ func queryRetry(
var maskErr error
switch analysis.API {
- case esparser.APIMaskSearch:
+ case parserbase.ElasticsearchAPIMaskSearch:
switch colName {
case "hits":
maskedValue, maskErr = maskElasticsearchHitsColumn(value, analysis.SortFields, objectSchema, semanticTypeToMaskerMap)
@@ -917,15 +875,15 @@ func queryRetry(
default:
continue
}
- case esparser.APIMaskGetDoc, esparser.APIMaskExplain:
+ case parserbase.ElasticsearchAPIMaskGetDoc, parserbase.ElasticsearchAPIMaskExplain:
if colName == "_source" {
maskedValue, maskErr = maskElasticsearchDocSource(value, objectSchema, semanticTypeToMaskerMap)
} else {
continue
}
- case esparser.APIMaskGetSource:
+ case parserbase.ElasticsearchAPIMaskGetSource:
maskedValue, maskErr = maskElasticsearchGetSourceColumn(colName, value, objectSchema, semanticTypeToMaskerMap)
- case esparser.APIMaskMGet:
+ case parserbase.ElasticsearchAPIMaskMGet:
if colName == "docs" {
maskedValue, maskErr = maskElasticsearchMGetSource(value, objectSchema, semanticTypeToMaskerMap)
} else {
diff --git a/backend/plugin/parser/base/document_analysis.go b/backend/plugin/parser/base/document_analysis.go
new file mode 100644
index 0000000000..985020ff25
--- /dev/null
+++ b/backend/plugin/parser/base/document_analysis.go
@@ -0,0 +1,77 @@
+package base
+
+// MongoDBMaskableAPI categorizes MongoDB collection reads for masking behavior.
+type MongoDBMaskableAPI int
+
+const (
+ MongoDBMaskableAPIUnsupported MongoDBMaskableAPI = iota
+ MongoDBMaskableAPIFind
+ MongoDBMaskableAPIFindOne
+ MongoDBMaskableAPIUnsupportedRead
+ MongoDBMaskableAPIAggregate
+)
+
+// MongoDBJoinedCollection records a $lookup or $graphLookup join extracted from an aggregate pipeline.
+type MongoDBJoinedCollection struct {
+ AsField string // The output field name added to each document (the "as" argument).
+ Collection string // The source collection being joined (the "from" argument).
+}
+
+// MongoDBAnalysis contains MongoDB statement data needed by masking checks.
+type MongoDBAnalysis struct {
+ API MongoDBMaskableAPI
+ Operation string
+ Collection string
+ PredicateFields []string
+ UnsupportedStage string // First pipeline stage that prevents masking. Only set when API == MongoDBMaskableAPIUnsupportedRead for aggregate pipelines.
+ JoinedCollections []MongoDBJoinedCollection // Join info extracted from $lookup and $graphLookup stages.
+}
+
+// ElasticsearchMaskableAPI categorizes ES endpoints for masking decisions.
+type ElasticsearchMaskableAPI int
+
+const (
+ ElasticsearchAPIUnsupported ElasticsearchMaskableAPI = iota
+ ElasticsearchAPIMaskSearch // covers _search and _msearch
+ ElasticsearchAPIMaskGetDoc // covers GET /<index>/_doc/<id>
+ ElasticsearchAPIMaskGetSource // covers GET /<index>/_source/<id>
+ ElasticsearchAPIMaskMGet // covers _mget
+ ElasticsearchAPIMaskExplain // covers _explain
+ ElasticsearchAPIBlocked // endpoint must be rejected when masking is active
+)
+
+// ElasticsearchBlockedFeature identifies a feature in the request body that must be blocked.
+type ElasticsearchBlockedFeature int
+
+const (
+ ElasticsearchBlockedFeatureAggs ElasticsearchBlockedFeature = iota
+ ElasticsearchBlockedFeatureSuggest
+ ElasticsearchBlockedFeatureScriptFields
+ ElasticsearchBlockedFeatureRuntimeMappings
+ ElasticsearchBlockedFeatureStoredFields
+ ElasticsearchBlockedFeatureDocvalueFields
+)
+
+// ElasticsearchBlockedFeatureNames maps ElasticsearchBlockedFeature to human-readable names for error messages.
+var ElasticsearchBlockedFeatureNames = map[ElasticsearchBlockedFeature]string{
+ ElasticsearchBlockedFeatureAggs: "aggregations",
+ ElasticsearchBlockedFeatureSuggest: "suggest",
+ ElasticsearchBlockedFeatureScriptFields: "script_fields",
+ ElasticsearchBlockedFeatureRuntimeMappings: "runtime_mappings",
+ ElasticsearchBlockedFeatureStoredFields: "stored_fields",
+ ElasticsearchBlockedFeatureDocvalueFields: "docvalue_fields",
+}
+
+// ElasticsearchAnalysis is the result of analyzing an ES REST request for masking.
+type ElasticsearchAnalysis struct {
+ API ElasticsearchMaskableAPI // Classification of the endpoint
+ Index string // Target index name extracted from URL (empty if not applicable)
+ BlockedFeatures []ElasticsearchBlockedFeature // Features found in request body that must be blocked
+ SourceFields []string // Fields requested via _source filtering (nil = all fields, empty = _source disabled)
+ SourceDisabled bool // True when _source: false is set
+ RequestedFields []string // Fields requested via "fields" parameter
+ HighlightFields []string // Fields requested via "highlight.fields"
+ SortFields []string // Field paths used in "sort"
+ HasInnerHits bool // True if query contains inner_hits
+ PredicateFields []string // Field paths used in query predicates
+}
diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.go
index 13e2377209..2b03cb4ed3 100644
--- a/backend/plugin/parser/base/span.go
+++ b/backend/plugin/parser/base/span.go
@@ -66,7 +66,11 @@ type QuerySpan struct {
// PredicatePaths stores the paths of fields used in query predicates for document databases
// (CosmosDB, MongoDB, Elasticsearch). For CosmosDB, paths begin with the container name.
// For MongoDB and Elasticsearch, paths are dot-delimited field names (e.g. "contact.phone").
- PredicatePaths map[string]*PathAST
+ PredicatePaths map[string]*PathAST
+ // MongoDBAnalysis caches the masking analysis for MongoDB statements, populated by GetQuerySpan.
+ MongoDBAnalysis *MongoDBAnalysis
+ // ElasticsearchAnalysis caches the masking analysis for Elasticsearch statements, populated by GetQuerySpan.
+ ElasticsearchAnalysis *ElasticsearchAnalysis
NotFoundError error
FunctionNotSupportedError error
}
diff --git a/backend/plugin/parser/elasticsearch/masking.go b/backend/plugin/parser/elasticsearch/masking.go
index acb3626af1..4631bcefb1 100644
--- a/backend/plugin/parser/elasticsearch/masking.go
+++ b/backend/plugin/parser/elasticsearch/masking.go
@@ -4,74 +4,39 @@ import (
"encoding/json"
"slices"
"strings"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
)
-// MaskableAPI categorizes ES endpoints for masking decisions.
-type MaskableAPI int
+// Type aliases so that internal code and external consumers (e.g. catalog_masking_elasticsearch.go)
+// can continue using the short names. The canonical definitions live in the base package.
+type (
+ MaskableAPI = base.ElasticsearchMaskableAPI
+ BlockedFeature = base.ElasticsearchBlockedFeature
+ RequestAnalysis = base.ElasticsearchAnalysis
+)
const (
- // APIUnsupported means the endpoint is not relevant to masking (no user data).
- APIUnsupported MaskableAPI = iota
- // APIMaskSearch covers _search and _msearch.
- APIMaskSearch
- // APIMaskGetDoc covers GET /<index>/_doc/<id>.
- APIMaskGetDoc
- // APIMaskGetSource covers GET /<index>/_source/<id>.
- APIMaskGetSource
- // APIMaskMGet covers _mget.
- APIMaskMGet
- // APIMaskExplain covers _explain.
- APIMaskExplain
- // APIBlocked means the endpoint must be rejected when masking is active.
- APIBlocked
+ APIUnsupported = base.ElasticsearchAPIUnsupported
+ APIMaskSearch = base.ElasticsearchAPIMaskSearch
+ APIMaskGetDoc = base.ElasticsearchAPIMaskGetDoc
+ APIMaskGetSource = base.ElasticsearchAPIMaskGetSource
+ APIMaskMGet = base.ElasticsearchAPIMaskMGet
+ APIMaskExplain = base.ElasticsearchAPIMaskExplain
+ APIBlocked = base.ElasticsearchAPIBlocked
)
-// BlockedFeature identifies a feature in the request body that must be blocked.
-type BlockedFeature int
-
const (
- BlockedFeatureAggs BlockedFeature = iota
- BlockedFeatureSuggest
- BlockedFeatureScriptFields
- BlockedFeatureRuntimeMappings
- BlockedFeatureStoredFields
- BlockedFeatureDocvalueFields
+ BlockedFeatureAggs = base.ElasticsearchBlockedFeatureAggs
+ BlockedFeatureSuggest = base.ElasticsearchBlockedFeatureSuggest
+ BlockedFeatureScriptFields = base.ElasticsearchBlockedFeatureScriptFields
+ BlockedFeatureRuntimeMappings = base.ElasticsearchBlockedFeatureRuntimeMappings
+ BlockedFeatureStoredFields = base.ElasticsearchBlockedFeatureStoredFields
+ BlockedFeatureDocvalueFields = base.ElasticsearchBlockedFeatureDocvalueFields
)
// BlockedFeatureNames maps BlockedFeature to human-readable names for error messages.
-var BlockedFeatureNames = map[BlockedFeature]string{
- BlockedFeatureAggs: "aggregations",
- BlockedFeatureSuggest: "suggest",
- BlockedFeatureScriptFields: "script_fields",
- BlockedFeatureRuntimeMappings: "runtime_mappings",
- BlockedFeatureStoredFields: "stored_fields",
- BlockedFeatureDocvalueFields: "docvalue_fields",
-}
-
-// RequestAnalysis is the result of analyzing an ES REST request for masking.
-type RequestAnalysis struct {
- // API is the classification of the endpoint.
- API MaskableAPI
- // Index is the target index name extracted from the URL (empty if not applicable).
- Index string
- // BlockedFeatures lists features found in the request body that must be blocked.
- BlockedFeatures []BlockedFeature
- // SourceFields is the list of fields requested via _source filtering (nil = all fields).
- // An empty slice means _source is disabled.
- SourceFields []string
- // SourceDisabled is true when _source: false is set.
- SourceDisabled bool
- // RequestedFields is the list of fields requested via the "fields" parameter.
- RequestedFields []string
- // HighlightFields is the list of fields requested via "highlight.fields".
- HighlightFields []string
- // SortFields is the list of field paths used in "sort".
- SortFields []string
- // HasInnerHits is true if the query contains inner_hits.
- HasInnerHits bool
- // PredicateFields is the list of field paths used in query predicates.
- PredicateFields []string
-}
+var BlockedFeatureNames = base.ElasticsearchBlockedFeatureNames
// blockedEndpoints lists URL patterns that must be rejected when masking is active.
// These overlap with read patterns but cannot be safely masked.
diff --git a/backend/plugin/parser/elasticsearch/query_span.go b/backend/plugin/parser/elasticsearch/query_span.go
index 35b1bd48af..9086878212 100644
--- a/backend/plugin/parser/elasticsearch/query_span.go
+++ b/backend/plugin/parser/elasticsearch/query_span.go
@@ -48,6 +48,8 @@ func GetQuerySpan(
span := &base.QuerySpan{Type: queryType}
analysis := AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))
+ span.ElasticsearchAnalysis = analysis
+
if len(analysis.PredicateFields) > 0 {
span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))
for _, field := range analysis.PredicateFields {
diff --git a/backend/plugin/parser/mongodb/masking.go b/backend/plugin/parser/mongodb/masking.go
index 6bed81e639..7d4e479d84 100644
--- a/backend/plugin/parser/mongodb/masking.go
+++ b/backend/plugin/parser/mongodb/masking.go
@@ -6,48 +6,28 @@ import (
"strings"
"github.com/antlr4-go/antlr/v4"
+ mongoparser "github.com/bytebase/parser/mongodb"
"github.com/pkg/errors"
- mongoparser "github.com/bytebase/parser/mongodb"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
)
-// MaskableAPI categorizes MongoDB collection reads for masking behavior.
-type MaskableAPI int
+// Type aliases so that internal code and external consumers (e.g. catalog_masking_mongodb.go)
+// can continue using the short names. The canonical definitions live in the base package.
+type (
+ MaskableAPI = base.MongoDBMaskableAPI
+ JoinedCollection = base.MongoDBJoinedCollection
+ MaskingAnalysis = base.MongoDBAnalysis
+)
const (
- // MaskableAPIUnsupported means the statement is not relevant for MongoDB masking.
- MaskableAPIUnsupported MaskableAPI = iota
- // MaskableAPIFind means db.<collection>.find(...).
- MaskableAPIFind
- // MaskableAPIFindOne means db.<collection>.findOne(...).
- MaskableAPIFindOne
- // MaskableAPIUnsupportedRead means a read API that is blocked in Milestone 1.
- MaskableAPIUnsupportedRead
- // MaskableAPIAggregate means db.<collection>.aggregate(...) with only shape-preserving stages.
- MaskableAPIAggregate
+ MaskableAPIUnsupported = base.MongoDBMaskableAPIUnsupported
+ MaskableAPIFind = base.MongoDBMaskableAPIFind
+ MaskableAPIFindOne = base.MongoDBMaskableAPIFindOne
+ MaskableAPIUnsupportedRead = base.MongoDBMaskableAPIUnsupportedRead
+ MaskableAPIAggregate = base.MongoDBMaskableAPIAggregate
)
-// JoinedCollection records a $lookup or $graphLookup join extracted from an aggregate pipeline.
-type JoinedCollection struct {
- // AsField is the output field name added to each document (the "as" argument).
- AsField string
- // Collection is the source collection being joined (the "from" argument).
- Collection string
-}
-
-// MaskingAnalysis contains MongoDB statement data needed by masking checks.
-type MaskingAnalysis struct {
- API MaskableAPI
- Operation string
- Collection string
- PredicateFields []string
- // UnsupportedStage is the first pipeline stage that prevents masking (e.g. "$group").
- // Only set when API == MaskableAPIUnsupportedRead for aggregate pipelines.
- UnsupportedStage string
- // JoinedCollections holds join info extracted from $lookup and $graphLookup stages.
- JoinedCollections []JoinedCollection
-}
-
// AnalyzeMaskingStatement analyzes a MongoDB shell statement for masking checks.
// It returns nil,nil for statements that are not relevant to masking flow.
func AnalyzeMaskingStatement(statement string) (*MaskingAnalysis, error) {
diff --git a/backend/plugin/parser/mongodb/query_span.go b/backend/plugin/parser/mongodb/query_span.go
index 9c550f24e6..abc22dccc1 100644
--- a/backend/plugin/parser/mongodb/query_span.go
+++ b/backend/plugin/parser/mongodb/query_span.go
@@ -29,6 +29,8 @@ func GetQuerySpan(_ context.Context, _ base.GetQuerySpanContext, stmt base.State
return span, nil
}
+ span.MongoDBAnalysis = analysis
+
if len(analysis.PredicateFields) > 0 {
span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))
for _, field := range analysis.PredicateFields {