Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(masking): support $unwind, $lookup, and $graphLookup in MongoDB aggregate masking (#19532)

h3n4l7dc1c4b09c02
diff --git a/backend/api/v1/catalog_masking.go b/backend/api/v1/catalog_masking.go
index e92eb0d559..0e6d936ca0 100644
--- a/backend/api/v1/catalog_masking.go
+++ b/backend/api/v1/catalog_masking.go
@@ -175,6 +175,17 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, sema
if objectSchema == nil {
return data, nil
}
+ // Schema is ARRAY but data is a single element (e.g. field unwound by $unwind).
+ // When there is no array-level semantic type, descend into the item schema so
+ // that item-level masking still applies to the unwound scalar or object.
+ if objectSchema.Type == storepb.ObjectSchema_ARRAY && objectSchema.SemanticType == "" {
+ if _, isArray := data.([]any); !isArray {
+ if itemSchema := objectSchema.GetArrayKind().GetKind(); itemSchema != nil {
+ return walkAndMaskJSONRecursive(data, itemSchema, semanticTypeToMasker)
+ }
+ return data, nil
+ }
+ }
switch data := data.(type) {
case map[string]any:
if objectSchema.SemanticType != "" {
diff --git a/backend/api/v1/catalog_masking_mongodb.go b/backend/api/v1/catalog_masking_mongodb.go
index c422782c66..35c6eb2031 100644
--- a/backend/api/v1/catalog_masking_mongodb.go
+++ b/backend/api/v1/catalog_masking_mongodb.go
@@ -2,6 +2,7 @@ package v1
import (
"encoding/json"
+ "maps"
"github.com/pkg/errors"
@@ -27,6 +28,51 @@ func checkMongoDBRequestBlocked(analysis *mongoparser.MaskingAnalysis) error {
return nil
}
+// injectJoinedSchemas adds array-of-objects schemas for each $lookup/$graphLookup join
+// into the source objectSchema so masking covers the joined field.
+// The as field holds an array of joined documents, so the injected schema is ARRAY{item: joinedSchema}.
+// If the source schema has no StructKind or the joined collection has no schema, the join is skipped silently.
+func injectJoinedSchemas(objectSchema *storepb.ObjectSchema, joins []joinedSchema) *storepb.ObjectSchema {
+ if objectSchema == nil || len(joins) == 0 {
+ return objectSchema
+ }
+ structKind := objectSchema.GetStructKind()
+ if structKind == nil {
+ return objectSchema
+ }
+
+ // Clone properties map so we don't mutate the cached schema.
+ props := make(map[string]*storepb.ObjectSchema, len(structKind.Properties)+len(joins))
+ maps.Copy(props, structKind.Properties)
+ for _, j := range joins {
+ if j.schema == nil {
+ continue
+ }
+ props[j.asField] = &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_ARRAY,
+ Kind: &storepb.ObjectSchema_ArrayKind_{
+ ArrayKind: &storepb.ObjectSchema_ArrayKind{
+ Kind: j.schema,
+ },
+ },
+ }
+ }
+ return &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: props,
+ },
+ },
+ }
+}
+
+// joinedSchema pairs an as-field name with its resolved ObjectSchema.
+type joinedSchema struct {
+ asField string
+ schema *storepb.ObjectSchema
+}
+
func maskMongoDBDocumentString(document string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {
if document == "" || objectSchema == nil {
return document, nil
diff --git a/backend/api/v1/catalog_masking_mongodb_test.go b/backend/api/v1/catalog_masking_mongodb_test.go
index 3fcb5ca6df..ea3a23b93d 100644
--- a/backend/api/v1/catalog_masking_mongodb_test.go
+++ b/backend/api/v1/catalog_masking_mongodb_test.go
@@ -147,6 +147,86 @@ func mustMongoSchema(t *testing.T, schemaName string) *storepb.ObjectSchema {
},
},
}
+ case "array-level-masked":
+ // tags array is masked at the array level (the whole array, or unwound scalar, is masked).
+ return &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "tags": {
+ Type: storepb.ObjectSchema_ARRAY,
+ SemanticType: "bb.default",
+ Kind: &storepb.ObjectSchema_ArrayKind_{
+ ArrayKind: &storepb.ObjectSchema_ArrayKind{
+ Kind: &storepb.ObjectSchema{Type: storepb.ObjectSchema_STRING},
+ },
+ },
+ },
+ "count": {Type: storepb.ObjectSchema_NUMBER},
+ },
+ },
+ },
+ }
+ case "array-of-objects":
+ // friends is an array of objects where phone is sensitive.
+ return &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "friends": {
+ Type: storepb.ObjectSchema_ARRAY,
+ Kind: &storepb.ObjectSchema_ArrayKind_{
+ ArrayKind: &storepb.ObjectSchema_ArrayKind{
+ Kind: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "name": {Type: storepb.ObjectSchema_STRING},
+ "phone": {Type: storepb.ObjectSchema_STRING, SemanticType: "bb.default"},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+ case "lookup-joined":
+ // users collection with an injected "orders" field (array of objects where total is sensitive).
+ return &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "name": {Type: storepb.ObjectSchema_STRING},
+ "orders": {
+ Type: storepb.ObjectSchema_ARRAY,
+ Kind: &storepb.ObjectSchema_ArrayKind_{
+ ArrayKind: &storepb.ObjectSchema_ArrayKind{
+ Kind: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "total": {Type: storepb.ObjectSchema_NUMBER, SemanticType: "bb.default"},
+ "status": {Type: storepb.ObjectSchema_STRING},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
case "non-object":
return &storepb.ObjectSchema{
Type: storepb.ObjectSchema_OBJECT,
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index 2e8271f900..4f95ac8d96 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -830,6 +830,18 @@ func queryRetry(
continue
}
+ if len(analysis.JoinedCollections) > 0 {
+ var joined []joinedSchema
+ for _, jc := range analysis.JoinedCollections {
+ js, jsErr := getMongoDBCollectionObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, jc.Collection)
+ if jsErr != nil {
+ return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.Wrapf(jsErr, "failed to get object schema for joined collection %q", jc.Collection))
+ }
+ joined = append(joined, joinedSchema{asField: jc.AsField, schema: js})
+ }
+ objectSchema = injectJoinedSchemas(objectSchema, joined)
+ }
+
for _, row := range result.Rows {
if len(row.Values) == 0 {
continue
diff --git a/backend/api/v1/test-data/mongodb_masking.yaml b/backend/api/v1/test-data/mongodb_masking.yaml
index 53ab7df1e0..d1c2bab229 100644
--- a/backend/api/v1/test-data/mongodb_masking.yaml
+++ b/backend/api/v1/test-data/mongodb_masking.yaml
@@ -24,6 +24,26 @@ maskDocument:
input: '"raw-string-result"'
want: '"raw-string-result"'
+ - description: unwind scalar - item-level semantic type masks unwound scalar
+ schema: array
+ input: '{"tags":"alpha","count":2}'
+ want: '{"tags":"******","count":2}'
+
+ - description: unwind scalar - array-level semantic type masks unwound scalar
+ schema: array-level-masked
+ input: '{"tags":"alpha","count":2}'
+ want: '{"tags":"******","count":2}'
+
+ - description: unwind object - item-level semantic type masks unwound object fields
+ schema: array-of-objects
+ input: '{"friends":{"name":"Bob","phone":"555-1234"}}'
+ want: '{"friends":{"name":"Bob","phone":"******"}}'
+
+ - description: lookup joined array field is masked
+ schema: lookup-joined
+ input: '{"name":"Alice","orders":[{"total":99.9,"status":"shipped"},{"total":5.0,"status":"pending"}]}'
+ want: '{"name":"Alice","orders":[{"total":"******","status":"shipped"},{"total":"******","status":"pending"}]}'
+
checkBlocked:
- description: nil analysis allowed
analysisNil: true
diff --git a/backend/plugin/parser/mongodb/masking.go b/backend/plugin/parser/mongodb/masking.go
index 69f7b31b0c..6bed81e639 100644
--- a/backend/plugin/parser/mongodb/masking.go
+++ b/backend/plugin/parser/mongodb/masking.go
@@ -27,6 +27,14 @@ const (
MaskableAPIAggregate
)
+// JoinedCollection records a $lookup or $graphLookup join extracted from an aggregate pipeline.
+type JoinedCollection struct {
+ // AsField is the output field name added to each document (the "as" argument).
+ AsField string
+ // Collection is the source collection being joined (the "from" argument).
+ Collection string
+}
+
// MaskingAnalysis contains MongoDB statement data needed by masking checks.
type MaskingAnalysis struct {
API MaskableAPI
@@ -36,6 +44,8 @@ type MaskingAnalysis struct {
// UnsupportedStage is the first pipeline stage that prevents masking (e.g. "$group").
// Only set when API == MaskableAPIUnsupportedRead for aggregate pipelines.
UnsupportedStage string
+ // JoinedCollections holds join info extracted from $lookup and $graphLookup stages.
+ JoinedCollections []JoinedCollection
}
// AnalyzeMaskingStatement analyzes a MongoDB shell statement for masking checks.
@@ -361,10 +371,11 @@ var shapePreservingAggregateStages = map[string]bool{
"$setWindowFields": true,
"$fill": true,
"$redact": true,
+ "$unwind": true,
}
func classifyAggregateForMasking(am mongoparser.IAggregateMethodContext) *MaskingAnalysis {
- predicateFields, unsupportedStage := extractAggregatePredicateFields(am.Arguments())
+ predicateFields, joinedCollections, unsupportedStage := extractAggregatePredicateFields(am.Arguments())
if unsupportedStage != "" {
return &MaskingAnalysis{
API: MaskableAPIUnsupportedRead,
@@ -373,61 +384,143 @@ func classifyAggregateForMasking(am mongoparser.IAggregateMethodContext) *Maskin
}
}
return &MaskingAnalysis{
- API: MaskableAPIAggregate,
- Operation: "aggregate",
- PredicateFields: predicateFields,
+ API: MaskableAPIAggregate,
+ Operation: "aggregate",
+ PredicateFields: predicateFields,
+ JoinedCollections: joinedCollections,
}
}
-// extractAggregatePredicateFields walks the pipeline and returns predicate fields from $match stages.
-// The second return value is the first unsupported stage name, or "" if all stages are allowed.
-func extractAggregatePredicateFields(args mongoparser.IArgumentsContext) ([]string, string) {
+// extractAggregatePredicateFields walks the pipeline and returns:
+// - predicate fields from $match stages
+// - join info from $lookup/$graphLookup stages
+// - the first unsupported stage name, or "" if all stages are allowed
+func extractAggregatePredicateFields(args mongoparser.IArgumentsContext) ([]string, []JoinedCollection, string) {
if args == nil {
- return nil, ""
+ return nil, nil, ""
}
allArgs := args.AllArgument()
if len(allArgs) == 0 {
- return nil, ""
+ return nil, nil, ""
}
first := allArgs[0]
if first == nil || first.Value() == nil {
- return nil, ""
+ return nil, nil, ""
}
arr := extractArrayValue(first.Value())
if arr == nil {
- return nil, "unknown"
+ return nil, nil, "unknown"
}
fields := make(map[string]struct{})
+ var joinedCollections []JoinedCollection
for _, elem := range arr.AllValue() {
doc := extractDocumentValue(elem)
if doc == nil {
- return nil, "unknown"
+ return nil, nil, "unknown"
}
pairs := doc.AllPair()
if len(pairs) == 0 {
continue
}
stageName := extractPairKey(pairs[0].Key())
- if !shapePreservingAggregateStages[stageName] {
- return nil, stageName
- }
- if stageName == "$match" {
- stageDoc := extractDocumentValue(pairs[0].Value())
- if stageDoc != nil {
- collectPredicateFieldsFromDocument(stageDoc, "", fields)
+ switch {
+ case shapePreservingAggregateStages[stageName]:
+ if stageName == "$match" {
+ stageDoc := extractDocumentValue(pairs[0].Value())
+ if stageDoc != nil {
+ collectPredicateFieldsFromDocument(stageDoc, "", fields)
+ }
+ }
+ case stageName == "$lookup":
+ join, unsupported := extractLookupJoin(pairs[0].Value())
+ if unsupported {
+ return nil, nil, "$lookup"
+ }
+ if join != nil {
+ joinedCollections = append(joinedCollections, *join)
}
+ case stageName == "$graphLookup":
+ join := extractGraphLookupJoin(pairs[0].Value())
+ if join != nil {
+ joinedCollections = append(joinedCollections, *join)
+ }
+ default:
+ return nil, nil, stageName
}
}
- if len(fields) == 0 {
- return nil, ""
+ var predicateFields []string
+ if len(fields) > 0 {
+ predicateFields = make([]string, 0, len(fields))
+ for field := range fields {
+ predicateFields = append(predicateFields, field)
+ }
}
- result := make([]string, 0, len(fields))
- for field := range fields {
- result = append(result, field)
+ return predicateFields, joinedCollections, ""
+}
+
+// extractLookupJoin parses a $lookup stage value and returns the join info.
+// Returns (nil, true) if this is a pipeline-form $lookup (unsupported).
+// Returns (nil, false) if from/as cannot be extracted (treated as passthrough).
+func extractLookupJoin(value mongoparser.IValueContext) (*JoinedCollection, bool) {
+ doc := extractDocumentValue(value)
+ if doc == nil {
+ return nil, false
+ }
+ var from, as string
+ for _, pair := range doc.AllPair() {
+ key := extractPairKey(pair.Key())
+ switch key {
+ case "pipeline":
+ // Pipeline form is not supported.
+ return nil, true
+ case "from":
+ from = extractStringLiteralValue(pair.Value())
+ case "as":
+ as = extractStringLiteralValue(pair.Value())
+ default:
+ }
+ }
+ if from == "" || as == "" {
+ return nil, false
}
- return result, ""
+ return &JoinedCollection{AsField: as, Collection: from}, false
+}
+
+// extractGraphLookupJoin parses a $graphLookup stage value and returns the join info.
+func extractGraphLookupJoin(value mongoparser.IValueContext) *JoinedCollection {
+ doc := extractDocumentValue(value)
+ if doc == nil {
+ return nil
+ }
+ var from, as string
+ for _, pair := range doc.AllPair() {
+ key := extractPairKey(pair.Key())
+ switch key {
+ case "from":
+ from = extractStringLiteralValue(pair.Value())
+ case "as":
+ as = extractStringLiteralValue(pair.Value())
+ default:
+ }
+ }
+ if from == "" || as == "" {
+ return nil
+ }
+ return &JoinedCollection{AsField: as, Collection: from}
+}
+
+// extractStringLiteralValue extracts a plain string value from a Value node.
+func extractStringLiteralValue(value mongoparser.IValueContext) string {
+ if value == nil {
+ return ""
+ }
+ text := value.GetText()
+ if len(text) >= 2 && (text[0] == '"' || text[0] == '\'') {
+ return unquoteMongoString(text)
+ }
+ return ""
}
diff --git a/backend/plugin/parser/mongodb/masking_test.go b/backend/plugin/parser/mongodb/masking_test.go
index c2dc6fe2a1..3f31a719cd 100644
--- a/backend/plugin/parser/mongodb/masking_test.go
+++ b/backend/plugin/parser/mongodb/masking_test.go
@@ -138,8 +138,20 @@ func TestAnalyzeMaskingStatement(t *testing.T) {
},
},
{
- description: "aggregate with $lookup is unsupported",
+ description: "aggregate with $lookup basic form is supported",
statement: `db.users.aggregate([{$lookup: {from: "orders", localField: "_id", foreignField: "userId", as: "orders"}}])`,
+ want: &MaskingAnalysis{
+ API: MaskableAPIAggregate,
+ Operation: "aggregate",
+ Collection: "users",
+ JoinedCollections: []JoinedCollection{
+ {AsField: "orders", Collection: "orders"},
+ },
+ },
+ },
+ {
+ description: "aggregate with $lookup pipeline form is unsupported",
+ statement: `db.users.aggregate([{$lookup: {from: "orders", pipeline: [{$match: {status: "active"}}], as: "orders"}}])`,
want: &MaskingAnalysis{
API: MaskableAPIUnsupportedRead,
Operation: "aggregate",
@@ -147,6 +159,18 @@ func TestAnalyzeMaskingStatement(t *testing.T) {
UnsupportedStage: "$lookup",
},
},
+ {
+ description: "aggregate with $graphLookup is supported",
+ statement: `db.users.aggregate([{$graphLookup: {from: "employees", startWith: "$reportsTo", connectFromField: "reportsTo", connectToField: "name", as: "reportingHierarchy"}}])`,
+ want: &MaskingAnalysis{
+ API: MaskableAPIAggregate,
+ Operation: "aggregate",
+ Collection: "users",
+ JoinedCollections: []JoinedCollection{
+ {AsField: "reportingHierarchy", Collection: "employees"},
+ },
+ },
+ },
{
description: "aggregate with $out is unsupported",
statement: `db.users.aggregate([{$match: {status: "active"}}, {$out: "activeUsers"}])`,
@@ -158,13 +182,22 @@ func TestAnalyzeMaskingStatement(t *testing.T) {
},
},
{
- description: "aggregate with $unwind is unsupported",
+ description: "aggregate with $unwind is shape-preserving",
statement: `db.users.aggregate([{$unwind: "$tags"}])`,
want: &MaskingAnalysis{
- API: MaskableAPIUnsupportedRead,
- Operation: "aggregate",
- Collection: "users",
- UnsupportedStage: "$unwind",
+ API: MaskableAPIAggregate,
+ Operation: "aggregate",
+ Collection: "users",
+ },
+ },
+ {
+ description: "aggregate with $match and $unwind",
+ statement: `db.users.aggregate([{$match: {status: "active"}}, {$unwind: "$tags"}])`,
+ want: &MaskingAnalysis{
+ API: MaskableAPIAggregate,
+ Operation: "aggregate",
+ Collection: "users",
+ PredicateFields: []string{"status"},
},
},
{