Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(masking): support $unwind, $lookup, and $graphLookup in MongoDB aggregate masking (#19532)
h3n4l7dc1c4b09c02
diff --git a/backend/api/v1/catalog_masking.go b/backend/api/v1/catalog_masking.goindex e92eb0d559..0e6d936ca0 100644--- a/backend/api/v1/catalog_masking.go+++ b/backend/api/v1/catalog_masking.go@@ -175,6 +175,17 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, semaif objectSchema == nil {return data, nil}+ // Schema is ARRAY but data is a single element (e.g. field unwound by $unwind).+ // When there is no array-level semantic type, descend into the item schema so+ // that item-level masking still applies to the unwound scalar or object.+ if objectSchema.Type == storepb.ObjectSchema_ARRAY && objectSchema.SemanticType == "" {+ if _, isArray := data.([]any); !isArray {+ if itemSchema := objectSchema.GetArrayKind().GetKind(); itemSchema != nil {+ return walkAndMaskJSONRecursive(data, itemSchema, semanticTypeToMasker)+ }+ return data, nil+ }+ }switch data := data.(type) {case map[string]any:if objectSchema.SemanticType != "" {diff --git a/backend/api/v1/catalog_masking_mongodb.go b/backend/api/v1/catalog_masking_mongodb.goindex c422782c66..35c6eb2031 100644--- a/backend/api/v1/catalog_masking_mongodb.go+++ b/backend/api/v1/catalog_masking_mongodb.go@@ -2,6 +2,7 @@ package v1import ("encoding/json"+ "maps""github.com/pkg/errors"@@ -27,6 +28,51 @@ func checkMongoDBRequestBlocked(analysis *mongoparser.MaskingAnalysis) error {return nil}+// injectJoinedSchemas adds array-of-objects schemas for each $lookup/$graphLookup join+// into the source objectSchema so masking covers the joined field.+// The as field holds an array of joined documents, so the injected schema is ARRAY{item: joinedSchema}.+// If the source schema has no StructKind or the joined collection has no schema, the join is skipped silently.+func injectJoinedSchemas(objectSchema *storepb.ObjectSchema, joins []joinedSchema) *storepb.ObjectSchema {+ if objectSchema == nil || len(joins) == 0 {+ return objectSchema+ }+ structKind := objectSchema.GetStructKind()+ if structKind == nil {+ return objectSchema+ }++ // Clone properties map so we don't mutate the cached schema.+ props := make(map[string]*storepb.ObjectSchema, len(structKind.Properties)+len(joins))+ maps.Copy(props, structKind.Properties)+ for _, j := range joins {+ if j.schema == nil {+ continue+ }+ props[j.asField] = &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_ARRAY,+ Kind: &storepb.ObjectSchema_ArrayKind_{+ ArrayKind: &storepb.ObjectSchema_ArrayKind{+ Kind: j.schema,+ },+ },+ }+ }+ return &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: props,+ },+ },+ }+}++// joinedSchema pairs an as-field name with its resolved ObjectSchema.+type joinedSchema struct {+ asField string+ schema *storepb.ObjectSchema+}+func maskMongoDBDocumentString(document string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {if document == "" || objectSchema == nil {return document, nildiff --git a/backend/api/v1/catalog_masking_mongodb_test.go b/backend/api/v1/catalog_masking_mongodb_test.goindex 3fcb5ca6df..ea3a23b93d 100644--- a/backend/api/v1/catalog_masking_mongodb_test.go+++ b/backend/api/v1/catalog_masking_mongodb_test.go@@ -147,6 +147,86 @@ func mustMongoSchema(t *testing.T, schemaName string) *storepb.ObjectSchema {},},}+ case "array-level-masked":+ // tags array is masked at the array level (the whole array, or unwound scalar, is masked).+ return &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "tags": {+ Type: storepb.ObjectSchema_ARRAY,+ SemanticType: "bb.default",+ Kind: &storepb.ObjectSchema_ArrayKind_{+ ArrayKind: &storepb.ObjectSchema_ArrayKind{+ Kind: &storepb.ObjectSchema{Type: storepb.ObjectSchema_STRING},+ },+ },+ },+ "count": {Type: storepb.ObjectSchema_NUMBER},+ },+ },+ },+ }+ case "array-of-objects":+ // friends is an array of objects where phone is sensitive.+ return &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "friends": {+ Type: storepb.ObjectSchema_ARRAY,+ Kind: &storepb.ObjectSchema_ArrayKind_{+ ArrayKind: &storepb.ObjectSchema_ArrayKind{+ Kind: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "name": {Type: storepb.ObjectSchema_STRING},+ "phone": {Type: storepb.ObjectSchema_STRING, SemanticType: "bb.default"},+ },+ },+ },+ },+ },+ },+ },+ },+ },+ },+ }+ case "lookup-joined":+ // users collection with an injected "orders" field (array of objects where total is sensitive).+ return &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "name": {Type: storepb.ObjectSchema_STRING},+ "orders": {+ Type: storepb.ObjectSchema_ARRAY,+ Kind: &storepb.ObjectSchema_ArrayKind_{+ ArrayKind: &storepb.ObjectSchema_ArrayKind{+ Kind: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "total": {Type: storepb.ObjectSchema_NUMBER, SemanticType: "bb.default"},+ "status": {Type: storepb.ObjectSchema_STRING},+ },+ },+ },+ },+ },+ },+ },+ },+ },+ },+ }case "non-object":return &storepb.ObjectSchema{Type: storepb.ObjectSchema_OBJECT,diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex 2e8271f900..4f95ac8d96 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -830,6 +830,18 @@ func queryRetry(continue}+ if len(analysis.JoinedCollections) > 0 {+ var joined []joinedSchema+ for _, jc := range analysis.JoinedCollections {+ js, jsErr := getMongoDBCollectionObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, jc.Collection)+ if jsErr != nil {+ return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.Wrapf(jsErr, "failed to get object schema for joined collection %q", jc.Collection))+ }+ joined = append(joined, joinedSchema{asField: jc.AsField, schema: js})+ }+ objectSchema = injectJoinedSchemas(objectSchema, joined)+ }+for _, row := range result.Rows {if len(row.Values) == 0 {continuediff --git a/backend/api/v1/test-data/mongodb_masking.yaml b/backend/api/v1/test-data/mongodb_masking.yamlindex 53ab7df1e0..d1c2bab229 100644--- a/backend/api/v1/test-data/mongodb_masking.yaml+++ b/backend/api/v1/test-data/mongodb_masking.yaml@@ -24,6 +24,26 @@ maskDocument:input: '"raw-string-result"'want: '"raw-string-result"'+ - description: unwind scalar - item-level semantic type masks unwound scalar+ schema: array+ input: '{"tags":"alpha","count":2}'+ want: '{"tags":"******","count":2}'++ - description: unwind scalar - array-level semantic type masks unwound scalar+ schema: array-level-masked+ input: '{"tags":"alpha","count":2}'+ want: '{"tags":"******","count":2}'++ - description: unwind object - item-level semantic type masks unwound object fields+ schema: array-of-objects+ input: '{"friends":{"name":"Bob","phone":"555-1234"}}'+ want: '{"friends":{"name":"Bob","phone":"******"}}'++ - description: lookup joined array field is masked+ schema: lookup-joined+ input: '{"name":"Alice","orders":[{"total":99.9,"status":"shipped"},{"total":5.0,"status":"pending"}]}'+ want: '{"name":"Alice","orders":[{"total":"******","status":"shipped"},{"total":"******","status":"pending"}]}'+checkBlocked:- description: nil analysis allowedanalysisNil: truediff --git a/backend/plugin/parser/mongodb/masking.go b/backend/plugin/parser/mongodb/masking.goindex 69f7b31b0c..6bed81e639 100644--- a/backend/plugin/parser/mongodb/masking.go+++ b/backend/plugin/parser/mongodb/masking.go@@ -27,6 +27,14 @@ const (MaskableAPIAggregate)+// JoinedCollection records a $lookup or $graphLookup join extracted from an aggregate pipeline.+type JoinedCollection struct {+ // AsField is the output field name added to each document (the "as" argument).+ AsField string+ // Collection is the source collection being joined (the "from" argument).+ Collection string+}+// MaskingAnalysis contains MongoDB statement data needed by masking checks.type MaskingAnalysis struct {API MaskableAPI@@ -36,6 +44,8 @@ type MaskingAnalysis struct {// UnsupportedStage is the first pipeline stage that prevents masking (e.g. "$group").// Only set when API == MaskableAPIUnsupportedRead for aggregate pipelines.UnsupportedStage string+ // JoinedCollections holds join info extracted from $lookup and $graphLookup stages.+ JoinedCollections []JoinedCollection}// AnalyzeMaskingStatement analyzes a MongoDB shell statement for masking checks.@@ -361,10 +371,11 @@ var shapePreservingAggregateStages = map[string]bool{"$setWindowFields": true,"$fill": true,"$redact": true,+ "$unwind": true,}func classifyAggregateForMasking(am mongoparser.IAggregateMethodContext) *MaskingAnalysis {- predicateFields, unsupportedStage := extractAggregatePredicateFields(am.Arguments())+ predicateFields, joinedCollections, unsupportedStage := extractAggregatePredicateFields(am.Arguments())if unsupportedStage != "" {return &MaskingAnalysis{API: MaskableAPIUnsupportedRead,@@ -373,61 +384,143 @@ func classifyAggregateForMasking(am mongoparser.IAggregateMethodContext) *Maskin}}return &MaskingAnalysis{- API: MaskableAPIAggregate,- Operation: "aggregate",- PredicateFields: predicateFields,+ API: MaskableAPIAggregate,+ Operation: "aggregate",+ PredicateFields: predicateFields,+ JoinedCollections: joinedCollections,}}-// extractAggregatePredicateFields walks the pipeline and returns predicate fields from $match stages.-// The second return value is the first unsupported stage name, or "" if all stages are allowed.-func extractAggregatePredicateFields(args mongoparser.IArgumentsContext) ([]string, string) {+// extractAggregatePredicateFields walks the pipeline and returns:+// - predicate fields from $match stages+// - join info from $lookup/$graphLookup stages+// - the first unsupported stage name, or "" if all stages are allowed+func extractAggregatePredicateFields(args mongoparser.IArgumentsContext) ([]string, []JoinedCollection, string) {if args == nil {- return nil, ""+ return nil, nil, ""}allArgs := args.AllArgument()if len(allArgs) == 0 {- return nil, ""+ return nil, nil, ""}first := allArgs[0]if first == nil || first.Value() == nil {- return nil, ""+ return nil, nil, ""}arr := extractArrayValue(first.Value())if arr == nil {- return nil, "unknown"+ return nil, nil, "unknown"}fields := make(map[string]struct{})+ var joinedCollections []JoinedCollectionfor _, elem := range arr.AllValue() {doc := extractDocumentValue(elem)if doc == nil {- return nil, "unknown"+ return nil, nil, "unknown"}pairs := doc.AllPair()if len(pairs) == 0 {continue}stageName := extractPairKey(pairs[0].Key())- if !shapePreservingAggregateStages[stageName] {- return nil, stageName- }- if stageName == "$match" {- stageDoc := extractDocumentValue(pairs[0].Value())- if stageDoc != nil {- collectPredicateFieldsFromDocument(stageDoc, "", fields)+ switch {+ case shapePreservingAggregateStages[stageName]:+ if stageName == "$match" {+ stageDoc := extractDocumentValue(pairs[0].Value())+ if stageDoc != nil {+ collectPredicateFieldsFromDocument(stageDoc, "", fields)+ }+ }+ case stageName == "$lookup":+ join, unsupported := extractLookupJoin(pairs[0].Value())+ if unsupported {+ return nil, nil, "$lookup"+ }+ if join != nil {+ joinedCollections = append(joinedCollections, *join)}+ case stageName == "$graphLookup":+ join := extractGraphLookupJoin(pairs[0].Value())+ if join != nil {+ joinedCollections = append(joinedCollections, *join)+ }+ default:+ return nil, nil, stageName}}- if len(fields) == 0 {- return nil, ""+ var predicateFields []string+ if len(fields) > 0 {+ predicateFields = make([]string, 0, len(fields))+ for field := range fields {+ predicateFields = append(predicateFields, field)+ }}- result := make([]string, 0, len(fields))- for field := range fields {- result = append(result, field)+ return predicateFields, joinedCollections, ""+}++// extractLookupJoin parses a $lookup stage value and returns the join info.+// Returns (nil, true) if this is a pipeline-form $lookup (unsupported).+// Returns (nil, false) if from/as cannot be extracted (treated as passthrough).+func extractLookupJoin(value mongoparser.IValueContext) (*JoinedCollection, bool) {+ doc := extractDocumentValue(value)+ if doc == nil {+ return nil, false+ }+ var from, as string+ for _, pair := range doc.AllPair() {+ key := extractPairKey(pair.Key())+ switch key {+ case "pipeline":+ // Pipeline form is not supported.+ return nil, true+ case "from":+ from = extractStringLiteralValue(pair.Value())+ case "as":+ as = extractStringLiteralValue(pair.Value())+ default:+ }+ }+ if from == "" || as == "" {+ return nil, false}- return result, ""+ return &JoinedCollection{AsField: as, Collection: from}, false+}++// extractGraphLookupJoin parses a $graphLookup stage value and returns the join info.+func extractGraphLookupJoin(value mongoparser.IValueContext) *JoinedCollection {+ doc := extractDocumentValue(value)+ if doc == nil {+ return nil+ }+ var from, as string+ for _, pair := range doc.AllPair() {+ key := extractPairKey(pair.Key())+ switch key {+ case "from":+ from = extractStringLiteralValue(pair.Value())+ case "as":+ as = extractStringLiteralValue(pair.Value())+ default:+ }+ }+ if from == "" || as == "" {+ return nil+ }+ return &JoinedCollection{AsField: as, Collection: from}+}++// extractStringLiteralValue extracts a plain string value from a Value node.+func extractStringLiteralValue(value mongoparser.IValueContext) string {+ if value == nil {+ return ""+ }+ text := value.GetText()+ if len(text) >= 2 && (text[0] == '"' || text[0] == '\'') {+ return unquoteMongoString(text)+ }+ return ""}diff --git a/backend/plugin/parser/mongodb/masking_test.go b/backend/plugin/parser/mongodb/masking_test.goindex c2dc6fe2a1..3f31a719cd 100644--- a/backend/plugin/parser/mongodb/masking_test.go+++ b/backend/plugin/parser/mongodb/masking_test.go@@ -138,8 +138,20 @@ func TestAnalyzeMaskingStatement(t *testing.T) {},},{- description: "aggregate with $lookup is unsupported",+ description: "aggregate with $lookup basic form is supported",statement: `db.users.aggregate([{$lookup: {from: "orders", localField: "_id", foreignField: "userId", as: "orders"}}])`,+ want: &MaskingAnalysis{+ API: MaskableAPIAggregate,+ Operation: "aggregate",+ Collection: "users",+ JoinedCollections: []JoinedCollection{+ {AsField: "orders", Collection: "orders"},+ },+ },+ },+ {+ description: "aggregate with $lookup pipeline form is unsupported",+ statement: `db.users.aggregate([{$lookup: {from: "orders", pipeline: [{$match: {status: "active"}}], as: "orders"}}])`,want: &MaskingAnalysis{API: MaskableAPIUnsupportedRead,Operation: "aggregate",@@ -147,6 +159,18 @@ func TestAnalyzeMaskingStatement(t *testing.T) {UnsupportedStage: "$lookup",},},+ {+ description: "aggregate with $graphLookup is supported",+ statement: `db.users.aggregate([{$graphLookup: {from: "employees", startWith: "$reportsTo", connectFromField: "reportsTo", connectToField: "name", as: "reportingHierarchy"}}])`,+ want: &MaskingAnalysis{+ API: MaskableAPIAggregate,+ Operation: "aggregate",+ Collection: "users",+ JoinedCollections: []JoinedCollection{+ {AsField: "reportingHierarchy", Collection: "employees"},+ },+ },+ },{description: "aggregate with $out is unsupported",statement: `db.users.aggregate([{$match: {status: "active"}}, {$out: "activeUsers"}])`,@@ -158,13 +182,22 @@ func TestAnalyzeMaskingStatement(t *testing.T) {},},{- description: "aggregate with $unwind is unsupported",+ description: "aggregate with $unwind is shape-preserving",statement: `db.users.aggregate([{$unwind: "$tags"}])`,want: &MaskingAnalysis{- API: MaskableAPIUnsupportedRead,- Operation: "aggregate",- Collection: "users",- UnsupportedStage: "$unwind",+ API: MaskableAPIAggregate,+ Operation: "aggregate",+ Collection: "users",+ },+ },+ {+ description: "aggregate with $match and $unwind",+ statement: `db.users.aggregate([{$match: {status: "active"}}, {$unwind: "$tags"}])`,+ want: &MaskingAnalysis{+ API: MaskableAPIAggregate,+ Operation: "aggregate",+ Collection: "users",+ PredicateFields: []string{"status"},},},{