Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: cache document DB analysis on QuerySpan and unify predicate checks (#19590)
h3n4l3f96d9002e88
diff --git a/backend/api/v1/catalog_masking.go b/backend/api/v1/catalog_masking.goindex 757bff7ef7..a31d55afdc 100644--- a/backend/api/v1/catalog_masking.go+++ b/backend/api/v1/catalog_masking.go@@ -1,23 +1,151 @@package v1import (+ "context""encoding/json"+ "fmt"+ "connectrpc.com/connect""github.com/pkg/errors""github.com/bytebase/bytebase/backend/component/masker"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"- "github.com/bytebase/bytebase/backend/plugin/parser/base"+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store")-func getFirstSemanticTypeInPath(ast *base.PathAST, objectSchema *storepb.ObjectSchema) string {+// preExecuteMaskingCheck runs engine-specific pre-execution checks that block+// queries with unsupported APIs. Sensitive predicate checks are handled+// post-execution on a per-statement basis by checkSensitivePredicates.+func preExecuteMaskingCheck(+ ctx context.Context,+ stores *store.Store,+ engine storepb.Engine,+ database *store.DatabaseMessage,+ spans []*parserbase.QuerySpan,+) error {+ switch engine {+ case storepb.Engine_MONGODB:+ return preExecuteMaskingCheckMongoDB(ctx, stores, database, spans)+ case storepb.Engine_ELASTICSEARCH:+ return preExecuteMaskingCheckElasticsearch(ctx, stores, database, spans)+ default:+ return nil+ }+}++func preExecuteMaskingCheckMongoDB(+ ctx context.Context,+ stores *store.Store,+ database *store.DatabaseMessage,+ spans []*parserbase.QuerySpan,+) error {+ for _, span := range spans {+ if span.MongoDBAnalysis == nil {+ continue+ }+ analysis := span.MongoDBAnalysis+ if analysis.Collection == "" {+ continue+ }++ objectSchema, err := getMongoDBCollectionObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, analysis.Collection)+ if err != nil {+ return connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get object schema for collection %q", analysis.Collection))+ }+ if objectSchema == nil {+ continue+ }++ if err := checkMongoDBRequestBlocked(analysis); err != nil {+ return connect.NewError(connect.CodeInvalidArgument, err)+ }+ }+ return nil+}++func preExecuteMaskingCheckElasticsearch(+ ctx context.Context,+ stores *store.Store,+ database *store.DatabaseMessage,+ spans []*parserbase.QuerySpan,+) error {+ for _, span := range spans {+ if span.ElasticsearchAnalysis == nil {+ continue+ }+ analysis := span.ElasticsearchAnalysis+ if analysis.API == parserbase.ElasticsearchAPIUnsupported {+ continue+ }+ indexName := analysis.Index+ if indexName == "" {+ continue+ }+ objectSchema, err := getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, indexName)+ if err != nil {+ return connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get object schema for index %q", indexName))+ }+ if objectSchema == nil || !objectSchemaHasSemanticTypes(objectSchema) {+ continue+ }+ if err := checkElasticsearchRequestBlocked(analysis); err != nil {+ return connect.NewError(connect.CodeInvalidArgument, err)+ }+ }+ return nil+}++// checkSensitivePredicates checks whether a span uses sensitive fields in+// predicates. Returns an error message if so, or "" if clean. This is used+// post-execution to blank individual statement results (matching SQL/CosmosDB+// behavior) rather than blocking the entire request.+func checkSensitivePredicates(+ ctx context.Context,+ stores *store.Store,+ database *store.DatabaseMessage,+ span *parserbase.QuerySpan,+) (string, error) {+ if len(span.PredicatePaths) == 0 {+ return "", nil+ }++ var objectSchema *storepb.ObjectSchema+ var err error++ if analysis := span.MongoDBAnalysis; analysis != nil && analysis.Collection != "" {+ objectSchema, err = getMongoDBCollectionObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, analysis.Collection)+ if err != nil {+ return "", errors.Wrapf(err, "failed to get object schema for collection %q", analysis.Collection)+ }+ } else if analysis := span.ElasticsearchAnalysis; analysis != nil && analysis.Index != "" {+ objectSchema, err = getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, analysis.Index)+ if err != nil {+ return "", errors.Wrapf(err, "failed to get object schema for index %q", analysis.Index)+ }+ }++ if objectSchema == nil {+ return "", nil+ }++ for pathStr := range span.PredicatePaths {+ semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)+ if semanticType != "" {+ return fmt.Sprintf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType), nil+ }+ }+ return "", nil+}++func getFirstSemanticTypeInPath(ast *parserbase.PathAST, objectSchema *storepb.ObjectSchema) string {if ast == nil || ast.Root == nil || objectSchema == nil {return ""}// Skip the first node because it always represents the container.- astWoutContainer := base.NewPathAST(ast.Root.GetNext())+ astWoutContainer := parserbase.NewPathAST(ast.Root.GetNext())if astWoutContainer == nil || astWoutContainer.Root == nil {return ""}@@ -34,7 +162,7 @@ func getFirstSemanticTypeInPath(ast *base.PathAST, objectSchema *storepb.ObjectS}switch node := node.(type) {- case *base.ItemSelector:+ case *parserbase.ItemSelector:if os.Type != storepb.ObjectSchema_OBJECT {return ""}@@ -48,7 +176,7 @@ func getFirstSemanticTypeInPath(ast *base.PathAST, objectSchema *storepb.ObjectSif !valid {return ""}- case *base.ArraySelector:+ case *parserbase.ArraySelector:if os.Type != storepb.ObjectSchema_OBJECT {return ""}@@ -82,30 +210,30 @@ func getFirstSemanticTypeInPath(ast *base.PathAST, objectSchema *storepb.ObjectSreturn ""}-func maskCosmosDB(span *base.QuerySpan, data map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {+func maskCosmosDB(span *parserbase.QuerySpan, data map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {if len(span.Results) != 1 {return nil, errors.Errorf("expected 1 result, but got %d", len(span.Results))}return walkAndMaskJSON(data, span.Results[0].SourceFieldPaths, objectSchema, semanticTypeToMasker)}-func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*base.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {+func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*parserbase.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {result := make(map[string]any)for key, value := range data {o := objectSchema- var ast *base.PathAST+ var ast *parserbase.PathASTif fieldPaths != nil {// Relocate the object schema cursor to the path position.// Skip the first node because it always represents the container.if path, ok := fieldPaths[key]; ok {if path != nil && path.Root != nil {- astWoutContainer := base.NewPathAST(path.Root.GetNext())+ astWoutContainer := parserbase.NewPathAST(path.Root.GetNext())ast = astWoutContainer}}}if ast == nil || ast.Root == nil {- ast = base.NewPathAST(base.NewItemSelector(key))+ ast = parserbase.NewPathAST(parserbase.NewItemSelector(key))}var parentSemanticType string@@ -130,7 +258,7 @@ func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*base.PathAST, oreturn result, nil}-func getObjectSchemaByPath(objectSchema *storepb.ObjectSchema, path *base.PathAST) (*storepb.ObjectSchema, string) {+func getObjectSchemaByPath(objectSchema *storepb.ObjectSchema, path *parserbase.PathAST) (*storepb.ObjectSchema, string) {outer := objectSchemaouterSemanticType := outer.SemanticTypeif outerSemanticType != "" {diff --git a/backend/api/v1/catalog_masking_elasticsearch.go b/backend/api/v1/catalog_masking_elasticsearch.goindex bbc49f56d4..d35e1ce817 100644--- a/backend/api/v1/catalog_masking_elasticsearch.go+++ b/backend/api/v1/catalog_masking_elasticsearch.go@@ -9,7 +9,7 @@ import ("github.com/bytebase/bytebase/backend/component/masker"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store")@@ -416,14 +416,14 @@ func maskElasticsearchMSearchSingleResponse(respMap map[string]any, sortFields [// checkElasticsearchRequestBlocked returns an error if the request must be blocked// because masking policies exist on the target index.-func checkElasticsearchRequestBlocked(analysis *esparser.RequestAnalysis) error {- if analysis.API == esparser.APIBlocked {+func checkElasticsearchRequestBlocked(analysis *parserbase.ElasticsearchAnalysis) error {+ if analysis.API == parserbase.ElasticsearchAPIBlocked {return errors.New("this Elasticsearch API is not supported when data masking is configured on the target index")}if len(analysis.BlockedFeatures) > 0 {names := make([]string, 0, len(analysis.BlockedFeatures))for _, f := range analysis.BlockedFeatures {- names = append(names, esparser.BlockedFeatureNames[f])+ names = append(names, parserbase.ElasticsearchBlockedFeatureNames[f])}return errors.Errorf("the following features are not supported when data masking is configured: %s", strings.Join(names, ", "))}diff --git a/backend/api/v1/catalog_masking_mongodb.go b/backend/api/v1/catalog_masking_mongodb.goindex 35c6eb2031..74c3a97359 100644--- a/backend/api/v1/catalog_masking_mongodb.go+++ b/backend/api/v1/catalog_masking_mongodb.go@@ -8,14 +8,14 @@ import ("github.com/bytebase/bytebase/backend/component/masker"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- mongoparser "github.com/bytebase/bytebase/backend/plugin/parser/mongodb"+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base")-func checkMongoDBRequestBlocked(analysis *mongoparser.MaskingAnalysis) error {+func checkMongoDBRequestBlocked(analysis *parserbase.MongoDBAnalysis) error {if analysis == nil {return nil}- if analysis.API == mongoparser.MaskableAPIUnsupportedRead {+ if analysis.API == parserbase.MongoDBMaskableAPIUnsupportedRead {operation := analysis.Operationif operation == "" {operation = "unknown"diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex 1c1410d5eb..36bcfb8d00 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -32,8 +32,6 @@ import ("github.com/bytebase/bytebase/backend/plugin/advisor/code""github.com/bytebase/bytebase/backend/plugin/db"parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"- esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"- mongoparser "github.com/bytebase/bytebase/backend/plugin/parser/mongodb""github.com/bytebase/bytebase/backend/plugin/schema""github.com/bytebase/bytebase/backend/runner/schemasync""github.com/bytebase/bytebase/backend/store"@@ -589,75 +587,9 @@ func queryRetry(maskingEnabled := !queryContext.Explain && !queryContext.SkipMasking &&licenseService.IsFeatureEnabledForInstance(v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil- // Pre-execution check for MongoDB: enforce Milestone 1 supported APIs and reject- // sensitive predicates before sending the query to the server.- if maskingEnabled && instance.Metadata.GetEngine() == storepb.Engine_MONGODB {- for i, stmt := range statements {- analysis, analyzeErr := mongoparser.AnalyzeMaskingStatement(stmt.Text)- if analyzeErr != nil || analysis == nil || analysis.Collection == "" {- continue- }-- objectSchema, schemaErr := getMongoDBCollectionObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, analysis.Collection)- if schemaErr != nil {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.Wrapf(schemaErr, "failed to get object schema for collection %q", analysis.Collection))- }- if objectSchema == nil {- continue- }-- if err := checkMongoDBRequestBlocked(analysis); err != nil {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)- }-- if i < len(spans) {- for pathStr := range spans[i].PredicatePaths {- semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)- if semanticType != "" {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,- errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType))- }- }- }- }- }-- // Pre-execution check for Elasticsearch: reject blocked APIs and predicate violations- // before sending the query to the server.- if maskingEnabled && instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH {- for i, stmt := range statements {- parsed, parseErr := esparser.ParseElasticsearchREST(stmt.Text)- if parseErr != nil || len(parsed.Requests) == 0 {- continue- }- req := parsed.Requests[0]- analysis := esparser.AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))- if analysis.API == esparser.APIUnsupported {- continue- }- indexName := analysis.Index- if indexName == "" {- continue- }- objectSchema, schemaErr := getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, indexName)- if schemaErr != nil {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.Wrapf(schemaErr, "failed to get object schema for index %q", indexName))- }- if objectSchema == nil || !objectSchemaHasSemanticTypes(objectSchema) {- continue- }- if err := checkElasticsearchRequestBlocked(analysis); err != nil {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)- }- if i < len(spans) {- for pathStr := range spans[i].PredicatePaths {- semanticType := lookupSemanticTypeByDotPath(pathStr, objectSchema)- if semanticType != "" {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,- errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", pathStr, semanticType))- }- }- }+ if maskingEnabled {+ if err := preExecuteMaskingCheck(ctx, stores, instance.Metadata.GetEngine(), database, spans); err != nil {+ return nil, nil, time.Duration(0), err}}@@ -813,12 +745,26 @@ func queryRetry(return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.New(err.Error()))}- for _, result := range results {- analysis, analyzeErr := mongoparser.AnalyzeMaskingStatement(result.Statement)- if analyzeErr != nil || analysis == nil || analysis.Collection == "" {+ for i, result := range results {+ if i < len(spans) {+ if errMsg, err := checkSensitivePredicates(ctx, stores, database, spans[i]); err != nil {+ return nil, nil, duration, connect.NewError(connect.CodeInternal, err)+ } else if errMsg != "" {+ result.Error = errMsg+ result.Rows = nil+ result.RowsCount = 0+ continue+ }+ }++ var analysis *parserbase.MongoDBAnalysis+ if i < len(spans) {+ analysis = spans[i].MongoDBAnalysis+ }+ if analysis == nil || analysis.Collection == "" {continue}- if analysis.API != mongoparser.MaskableAPIFind && analysis.API != mongoparser.MaskableAPIFindOne && analysis.API != mongoparser.MaskableAPIAggregate {+ if analysis.API != parserbase.MongoDBMaskableAPIFind && analysis.API != parserbase.MongoDBMaskableAPIFindOne && analysis.API != parserbase.MongoDBMaskableAPIAggregate {continue}@@ -864,15 +810,27 @@ func queryRetry(}}} else if instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH {- for _, result := range results {- parsed, err := esparser.ParseElasticsearchREST(result.Statement)- if err != nil || len(parsed.Requests) == 0 {+ for i, result := range results {+ if i < len(spans) {+ if errMsg, err := checkSensitivePredicates(ctx, stores, database, spans[i]); err != nil {+ return nil, nil, duration, connect.NewError(connect.CodeInternal, err)+ } else if errMsg != "" {+ result.Error = errMsg+ result.Rows = nil+ result.RowsCount = 0+ continue+ }+ }++ var analysis *parserbase.ElasticsearchAnalysis+ if i < len(spans) {+ analysis = spans[i].ElasticsearchAnalysis+ }+ if analysis == nil {continue}- req := parsed.Requests[0]- analysis := esparser.AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))- if analysis.API == esparser.APIUnsupported || analysis.API == esparser.APIBlocked {+ if analysis.API == parserbase.ElasticsearchAPIUnsupported || analysis.API == parserbase.ElasticsearchAPIBlocked {continue}@@ -908,7 +866,7 @@ func queryRetry(var maskErr errorswitch analysis.API {- case esparser.APIMaskSearch:+ case parserbase.ElasticsearchAPIMaskSearch:switch colName {case "hits":maskedValue, maskErr = maskElasticsearchHitsColumn(value, analysis.SortFields, objectSchema, semanticTypeToMaskerMap)@@ -917,15 +875,15 @@ func queryRetry(default:continue}- case esparser.APIMaskGetDoc, esparser.APIMaskExplain:+ case parserbase.ElasticsearchAPIMaskGetDoc, parserbase.ElasticsearchAPIMaskExplain:if colName == "_source" {maskedValue, maskErr = maskElasticsearchDocSource(value, objectSchema, semanticTypeToMaskerMap)} else {continue}- case esparser.APIMaskGetSource:+ case parserbase.ElasticsearchAPIMaskGetSource:maskedValue, maskErr = maskElasticsearchGetSourceColumn(colName, value, objectSchema, semanticTypeToMaskerMap)- case esparser.APIMaskMGet:+ case parserbase.ElasticsearchAPIMaskMGet:if colName == "docs" {maskedValue, maskErr = maskElasticsearchMGetSource(value, objectSchema, semanticTypeToMaskerMap)} else {diff --git a/backend/plugin/parser/base/document_analysis.go b/backend/plugin/parser/base/document_analysis.gonew file mode 100644index 0000000000..985020ff25--- /dev/null+++ b/backend/plugin/parser/base/document_analysis.go@@ -0,0 +1,77 @@+package base++// MongoDBMaskableAPI categorizes MongoDB collection reads for masking behavior.+type MongoDBMaskableAPI int++const (+ MongoDBMaskableAPIUnsupported MongoDBMaskableAPI = iota+ MongoDBMaskableAPIFind+ MongoDBMaskableAPIFindOne+ MongoDBMaskableAPIUnsupportedRead+ MongoDBMaskableAPIAggregate+)++// MongoDBJoinedCollection records a $lookup or $graphLookup join extracted from an aggregate pipeline.+type MongoDBJoinedCollection struct {+ AsField string // The output field name added to each document (the "as" argument).+ Collection string // The source collection being joined (the "from" argument).+}++// MongoDBAnalysis contains MongoDB statement data needed by masking checks.+type MongoDBAnalysis struct {+ API MongoDBMaskableAPI+ Operation string+ Collection string+ PredicateFields []string+ UnsupportedStage string // First pipeline stage that prevents masking. Only set when API == MongoDBMaskableAPIUnsupportedRead for aggregate pipelines.+ JoinedCollections []MongoDBJoinedCollection // Join info extracted from $lookup and $graphLookup stages.+}++// ElasticsearchMaskableAPI categorizes ES endpoints for masking decisions.+type ElasticsearchMaskableAPI int++const (+ ElasticsearchAPIUnsupported ElasticsearchMaskableAPI = iota+ ElasticsearchAPIMaskSearch // covers _search and _msearch+ ElasticsearchAPIMaskGetDoc // covers GET /<index>/_doc/<id>+ ElasticsearchAPIMaskGetSource // covers GET /<index>/_source/<id>+ ElasticsearchAPIMaskMGet // covers _mget+ ElasticsearchAPIMaskExplain // covers _explain+ ElasticsearchAPIBlocked // endpoint must be rejected when masking is active+)++// ElasticsearchBlockedFeature identifies a feature in the request body that must be blocked.+type ElasticsearchBlockedFeature int++const (+ ElasticsearchBlockedFeatureAggs ElasticsearchBlockedFeature = iota+ ElasticsearchBlockedFeatureSuggest+ ElasticsearchBlockedFeatureScriptFields+ ElasticsearchBlockedFeatureRuntimeMappings+ ElasticsearchBlockedFeatureStoredFields+ ElasticsearchBlockedFeatureDocvalueFields+)++// ElasticsearchBlockedFeatureNames maps ElasticsearchBlockedFeature to human-readable names for error messages.+var ElasticsearchBlockedFeatureNames = map[ElasticsearchBlockedFeature]string{+ ElasticsearchBlockedFeatureAggs: "aggregations",+ ElasticsearchBlockedFeatureSuggest: "suggest",+ ElasticsearchBlockedFeatureScriptFields: "script_fields",+ ElasticsearchBlockedFeatureRuntimeMappings: "runtime_mappings",+ ElasticsearchBlockedFeatureStoredFields: "stored_fields",+ ElasticsearchBlockedFeatureDocvalueFields: "docvalue_fields",+}++// ElasticsearchAnalysis is the result of analyzing an ES REST request for masking.+type ElasticsearchAnalysis struct {+ API ElasticsearchMaskableAPI // Classification of the endpoint+ Index string // Target index name extracted from URL (empty if not applicable)+ BlockedFeatures []ElasticsearchBlockedFeature // Features found in request body that must be blocked+ SourceFields []string // Fields requested via _source filtering (nil = all fields, empty = _source disabled)+ SourceDisabled bool // True when _source: false is set+ RequestedFields []string // Fields requested via "fields" parameter+ HighlightFields []string // Fields requested via "highlight.fields"+ SortFields []string // Field paths used in "sort"+ HasInnerHits bool // True if query contains inner_hits+ PredicateFields []string // Field paths used in query predicates+}diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.goindex 13e2377209..2b03cb4ed3 100644--- a/backend/plugin/parser/base/span.go+++ b/backend/plugin/parser/base/span.go@@ -66,7 +66,11 @@ type QuerySpan struct {// PredicatePaths stores the paths of fields used in query predicates for document databases// (CosmosDB, MongoDB, Elasticsearch). For CosmosDB, paths begin with the container name.// For MongoDB and Elasticsearch, paths are dot-delimited field names (e.g. "contact.phone").- PredicatePaths map[string]*PathAST+ PredicatePaths map[string]*PathAST+ // MongoDBAnalysis caches the masking analysis for MongoDB statements, populated by GetQuerySpan.+ MongoDBAnalysis *MongoDBAnalysis+ // ElasticsearchAnalysis caches the masking analysis for Elasticsearch statements, populated by GetQuerySpan.+ ElasticsearchAnalysis *ElasticsearchAnalysisNotFoundError errorFunctionNotSupportedError error}diff --git a/backend/plugin/parser/elasticsearch/masking.go b/backend/plugin/parser/elasticsearch/masking.goindex acb3626af1..4631bcefb1 100644--- a/backend/plugin/parser/elasticsearch/masking.go+++ b/backend/plugin/parser/elasticsearch/masking.go@@ -4,74 +4,39 @@ import ("encoding/json""slices""strings"++ "github.com/bytebase/bytebase/backend/plugin/parser/base")-// MaskableAPI categorizes ES endpoints for masking decisions.-type MaskableAPI int+// Type aliases so that internal code and external consumers (e.g. catalog_masking_elasticsearch.go)+// can continue using the short names. The canonical definitions live in the base package.+type (+ MaskableAPI = base.ElasticsearchMaskableAPI+ BlockedFeature = base.ElasticsearchBlockedFeature+ RequestAnalysis = base.ElasticsearchAnalysis+)const (- // APIUnsupported means the endpoint is not relevant to masking (no user data).- APIUnsupported MaskableAPI = iota- // APIMaskSearch covers _search and _msearch.- APIMaskSearch- // APIMaskGetDoc covers GET /<index>/_doc/<id>.- APIMaskGetDoc- // APIMaskGetSource covers GET /<index>/_source/<id>.- APIMaskGetSource- // APIMaskMGet covers _mget.- APIMaskMGet- // APIMaskExplain covers _explain.- APIMaskExplain- // APIBlocked means the endpoint must be rejected when masking is active.- APIBlocked+ APIUnsupported = base.ElasticsearchAPIUnsupported+ APIMaskSearch = base.ElasticsearchAPIMaskSearch+ APIMaskGetDoc = base.ElasticsearchAPIMaskGetDoc+ APIMaskGetSource = base.ElasticsearchAPIMaskGetSource+ APIMaskMGet = base.ElasticsearchAPIMaskMGet+ APIMaskExplain = base.ElasticsearchAPIMaskExplain+ APIBlocked = base.ElasticsearchAPIBlocked)-// BlockedFeature identifies a feature in the request body that must be blocked.-type BlockedFeature int-const (- BlockedFeatureAggs BlockedFeature = iota- BlockedFeatureSuggest- BlockedFeatureScriptFields- BlockedFeatureRuntimeMappings- BlockedFeatureStoredFields- BlockedFeatureDocvalueFields+ BlockedFeatureAggs = base.ElasticsearchBlockedFeatureAggs+ BlockedFeatureSuggest = base.ElasticsearchBlockedFeatureSuggest+ BlockedFeatureScriptFields = base.ElasticsearchBlockedFeatureScriptFields+ BlockedFeatureRuntimeMappings = base.ElasticsearchBlockedFeatureRuntimeMappings+ BlockedFeatureStoredFields = base.ElasticsearchBlockedFeatureStoredFields+ BlockedFeatureDocvalueFields = base.ElasticsearchBlockedFeatureDocvalueFields)// BlockedFeatureNames maps BlockedFeature to human-readable names for error messages.-var BlockedFeatureNames = map[BlockedFeature]string{- BlockedFeatureAggs: "aggregations",- BlockedFeatureSuggest: "suggest",- BlockedFeatureScriptFields: "script_fields",- BlockedFeatureRuntimeMappings: "runtime_mappings",- BlockedFeatureStoredFields: "stored_fields",- BlockedFeatureDocvalueFields: "docvalue_fields",-}--// RequestAnalysis is the result of analyzing an ES REST request for masking.-type RequestAnalysis struct {- // API is the classification of the endpoint.- API MaskableAPI- // Index is the target index name extracted from the URL (empty if not applicable).- Index string- // BlockedFeatures lists features found in the request body that must be blocked.- BlockedFeatures []BlockedFeature- // SourceFields is the list of fields requested via _source filtering (nil = all fields).- // An empty slice means _source is disabled.- SourceFields []string- // SourceDisabled is true when _source: false is set.- SourceDisabled bool- // RequestedFields is the list of fields requested via the "fields" parameter.- RequestedFields []string- // HighlightFields is the list of fields requested via "highlight.fields".- HighlightFields []string- // SortFields is the list of field paths used in "sort".- SortFields []string- // HasInnerHits is true if the query contains inner_hits.- HasInnerHits bool- // PredicateFields is the list of field paths used in query predicates.- PredicateFields []string-}+var BlockedFeatureNames = base.ElasticsearchBlockedFeatureNames// blockedEndpoints lists URL patterns that must be rejected when masking is active.// These overlap with read patterns but cannot be safely masked.diff --git a/backend/plugin/parser/elasticsearch/query_span.go b/backend/plugin/parser/elasticsearch/query_span.goindex 35b1bd48af..9086878212 100644--- a/backend/plugin/parser/elasticsearch/query_span.go+++ b/backend/plugin/parser/elasticsearch/query_span.go@@ -48,6 +48,8 @@ func GetQuerySpan(span := &base.QuerySpan{Type: queryType}analysis := AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))+ span.ElasticsearchAnalysis = analysis+if len(analysis.PredicateFields) > 0 {span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))for _, field := range analysis.PredicateFields {diff --git a/backend/plugin/parser/mongodb/masking.go b/backend/plugin/parser/mongodb/masking.goindex 6bed81e639..7d4e479d84 100644--- a/backend/plugin/parser/mongodb/masking.go+++ b/backend/plugin/parser/mongodb/masking.go@@ -6,48 +6,28 @@ import ("strings""github.com/antlr4-go/antlr/v4"+ mongoparser "github.com/bytebase/parser/mongodb""github.com/pkg/errors"- mongoparser "github.com/bytebase/parser/mongodb"+ "github.com/bytebase/bytebase/backend/plugin/parser/base")-// MaskableAPI categorizes MongoDB collection reads for masking behavior.-type MaskableAPI int+// Type aliases so that internal code and external consumers (e.g. catalog_masking_mongodb.go)+// can continue using the short names. The canonical definitions live in the base package.+type (+ MaskableAPI = base.MongoDBMaskableAPI+ JoinedCollection = base.MongoDBJoinedCollection+ MaskingAnalysis = base.MongoDBAnalysis+)const (- // MaskableAPIUnsupported means the statement is not relevant for MongoDB masking.- MaskableAPIUnsupported MaskableAPI = iota- // MaskableAPIFind means db.<collection>.find(...).- MaskableAPIFind- // MaskableAPIFindOne means db.<collection>.findOne(...).- MaskableAPIFindOne- // MaskableAPIUnsupportedRead means a read API that is blocked in Milestone 1.- MaskableAPIUnsupportedRead- // MaskableAPIAggregate means db.<collection>.aggregate(...) with only shape-preserving stages.- MaskableAPIAggregate+ MaskableAPIUnsupported = base.MongoDBMaskableAPIUnsupported+ MaskableAPIFind = base.MongoDBMaskableAPIFind+ MaskableAPIFindOne = base.MongoDBMaskableAPIFindOne+ MaskableAPIUnsupportedRead = base.MongoDBMaskableAPIUnsupportedRead+ MaskableAPIAggregate = base.MongoDBMaskableAPIAggregate)-// JoinedCollection records a $lookup or $graphLookup join extracted from an aggregate pipeline.-type JoinedCollection struct {- // AsField is the output field name added to each document (the "as" argument).- AsField string- // Collection is the source collection being joined (the "from" argument).- Collection string-}--// MaskingAnalysis contains MongoDB statement data needed by masking checks.-type MaskingAnalysis struct {- API MaskableAPI- Operation string- Collection string- PredicateFields []string- // UnsupportedStage is the first pipeline stage that prevents masking (e.g. "$group").- // Only set when API == MaskableAPIUnsupportedRead for aggregate pipelines.- UnsupportedStage string- // JoinedCollections holds join info extracted from $lookup and $graphLookup stages.- JoinedCollections []JoinedCollection-}-// AnalyzeMaskingStatement analyzes a MongoDB shell statement for masking checks.// It returns nil,nil for statements that are not relevant to masking flow.func AnalyzeMaskingStatement(statement string) (*MaskingAnalysis, error) {diff --git a/backend/plugin/parser/mongodb/query_span.go b/backend/plugin/parser/mongodb/query_span.goindex 9c550f24e6..abc22dccc1 100644--- a/backend/plugin/parser/mongodb/query_span.go+++ b/backend/plugin/parser/mongodb/query_span.go@@ -29,6 +29,8 @@ func GetQuerySpan(_ context.Context, _ base.GetQuerySpanContext, stmt base.Statereturn span, nil}+ span.MongoDBAnalysis = analysis+if len(analysis.PredicateFields) > 0 {span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))for _, field := range analysis.PredicateFields {