Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

Fix semantic JSON masking for non-full maskers (#19540)

h3n4lb4ec1e6908a2
diff --git a/backend/api/v1/catalog_masking.go b/backend/api/v1/catalog_masking.go
index 0e6d936ca0..757bff7ef7 100644
--- a/backend/api/v1/catalog_masking.go
+++ b/backend/api/v1/catalog_masking.go
@@ -1,6 +1,8 @@
package v1
import (
+ "encoding/json"
+
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/component/masker"
@@ -110,7 +112,11 @@ func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*base.PathAST, o
o, parentSemanticType = getObjectSchemaByPath(o, ast)
if parentSemanticType != "" {
if m, ok := semanticTypeToMasker[parentSemanticType]; ok {
- result[key] = getJSONMemberFromRowValue(m.Mask(nil))
+ maskedValue, err := applyMaskerToJSONMember(value, m)
+ if err != nil {
+ return nil, err
+ }
+ result[key] = maskedValue
continue
}
}
@@ -191,7 +197,7 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, sema
if objectSchema.SemanticType != "" {
// If the semantic type is found, replace the entire value directly.
if m, ok := semanticTypeToMasker[objectSchema.SemanticType]; ok {
- return getJSONMemberFromRowValue(m.Mask(nil)), nil
+ return applyMaskerToJSONMember(data, m)
}
} else {
// Otherwise, recursively walk the object.
@@ -216,7 +222,7 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, sema
if objectSchema.SemanticType != "" {
// If the semantic type is found, replace the entire value directly.
if m, ok := semanticTypeToMasker[objectSchema.SemanticType]; ok {
- return getJSONMemberFromRowValue(m.Mask(nil)), nil
+ return applyMaskerToJSONMember(data, m)
}
} else {
arrayKind := objectSchema.GetArrayKind()
@@ -252,6 +258,24 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, sema
return data, nil
}
+func applyMaskerToJSONMember(data any, m masker.Masker) (any, error) {
+ if rowValue, ok := getRowValueFromJSONAtomicMember(data); ok {
+ return getJSONMemberFromRowValue(m.Mask(&masker.MaskData{Data: rowValue})), nil
+ }
+
+ jsonBytes, err := json.Marshal(data)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to marshal JSON member")
+ }
+ return getJSONMemberFromRowValue(m.Mask(&masker.MaskData{
+ Data: &v1pb.RowValue{
+ Kind: &v1pb.RowValue_StringValue{
+ StringValue: string(jsonBytes),
+ },
+ },
+ })), nil
+}
+
func applyMaskerToData(data any, m masker.Masker) (any, error) {
switch data := data.(type) {
case map[string]any:
@@ -285,7 +309,8 @@ func applyMaskerToData(data any, m masker.Masker) (any, error) {
func getJSONMemberFromRowValue(rowValue *v1pb.RowValue) any {
switch rowValue := rowValue.Kind.(type) {
- // TODO: Handle NULL, VALUE_VALUE, TIMESTAMP_VALUE, TIMESTAMPTZVALUE.
+ case *v1pb.RowValue_NullValue:
+ return nil
case *v1pb.RowValue_BoolValue:
return rowValue.BoolValue
case *v1pb.RowValue_BytesValue:
@@ -327,6 +352,5 @@ func getRowValueFromJSONAtomicMember(data any) (result *v1pb.RowValue, ok bool)
Kind: &v1pb.RowValue_BoolValue{BoolValue: data},
}, true
}
- // TODO: Handle NULL.
return nil, false
}
diff --git a/backend/api/v1/catalog_masking_test.go b/backend/api/v1/catalog_masking_test.go
index 200e176c00..d04d91df0f 100644
--- a/backend/api/v1/catalog_masking_test.go
+++ b/backend/api/v1/catalog_masking_test.go
@@ -135,6 +135,9 @@ func TestGetFirstSemanticTypeInPath(t *testing.T) {
}
func TestWalkAndMaskJSON(t *testing.T) {
+ innerMasker, err := masker.NewInnerOuterMasker(storepb.Algorithm_InnerOuterMask_INNER, 1, 1, "*")
+ require.NoError(t, err)
+
type testCase struct {
description string
input string
@@ -319,6 +322,63 @@ func TestWalkAndMaskJSON(t *testing.T) {
},
want: `{"name": "John", "tags": "******"}`,
},
+ {
+ description: "tagged field with object value is masked as opaque JSON string for non-full masker",
+ input: `{"name": "John", "address": {"street": "123 Main St", "city": "NYC"}}`,
+ objectSchema: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "address": {
+ Type: storepb.ObjectSchema_OBJECT,
+ SemanticType: "PII",
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "street": {Type: storepb.ObjectSchema_STRING},
+ "city": {Type: storepb.ObjectSchema_STRING},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ semanticTypeToMasker: map[string]masker.Masker{
+ "PII": innerMasker,
+ },
+ want: `{"name": "John", "address": "{***********************************}"}`,
+ },
+ {
+ description: "tagged field with array value is masked as opaque JSON string for non-full masker",
+ input: `{"name": "John", "tags": ["tag1", "tag2"]}`,
+ objectSchema: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "tags": {
+ Type: storepb.ObjectSchema_ARRAY,
+ SemanticType: "PII",
+ Kind: &storepb.ObjectSchema_ArrayKind_{
+ ArrayKind: &storepb.ObjectSchema_ArrayKind{
+ Kind: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_STRING,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ semanticTypeToMasker: map[string]masker.Masker{
+ "PII": innerMasker,
+ },
+ want: `{"name": "John", "tags": "[*************]"}`,
+ },
{
description: "mask following the field paths",
input: `{
@@ -406,3 +466,65 @@ func TestWalkAndMaskJSON(t *testing.T) {
require.JSONEqf(t, tc.want, string(output), tc.description)
}
}
+
+func TestWalkAndMaskJSONRecursiveSemanticTypeMasksJSONContainerAsString(t *testing.T) {
+ innerMasker, err := masker.NewInnerOuterMasker(storepb.Algorithm_InnerOuterMask_INNER, 1, 1, "*")
+ require.NoError(t, err)
+
+ testCases := []struct {
+ description string
+ input string
+ objectSchema *storepb.ObjectSchema
+ want string
+ }{
+ {
+ description: "root object semantic type is masked as opaque JSON string",
+ input: `{"street": "123 Main St", "city": "NYC"}`,
+ objectSchema: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ SemanticType: "PII",
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "street": {Type: storepb.ObjectSchema_STRING},
+ "city": {Type: storepb.ObjectSchema_STRING},
+ },
+ },
+ },
+ },
+ want: `"{***********************************}"`,
+ },
+ {
+ description: "root array semantic type is masked as opaque JSON string",
+ input: `["tag1", "tag2"]`,
+ objectSchema: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_ARRAY,
+ SemanticType: "PII",
+ Kind: &storepb.ObjectSchema_ArrayKind_{
+ ArrayKind: &storepb.ObjectSchema_ArrayKind{
+ Kind: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_STRING,
+ },
+ },
+ },
+ },
+ want: `"[*************]"`,
+ },
+ }
+
+ for _, tc := range testCases {
+ var input any
+ err := json.Unmarshal([]byte(tc.input), &input)
+ require.NoError(t, err, tc.description)
+
+ got, err := walkAndMaskJSONRecursive(input, tc.objectSchema, map[string]masker.Masker{
+ "PII": innerMasker,
+ })
+ require.NoError(t, err, tc.description)
+
+ output, err := json.Marshal(got)
+ require.NoError(t, err, tc.description)
+
+ require.JSONEqf(t, tc.want, string(output), tc.description)
+ }
+}
diff --git a/backend/component/masker/masker.go b/backend/component/masker/masker.go
index bacc6fd11b..162939c4db 100644
--- a/backend/component/masker/masker.go
+++ b/backend/component/masker/masker.go
@@ -620,7 +620,7 @@ func maskProtoValue(m Masker, value *structpb.Value) *structpb.Value {
//nolint
f := v.Kind.(*structpb.Value_StructValue)
for field, value := range f.StructValue.Fields {
- kindValue.StructValue.Fields[field] = maskProtoValue(m, value)
+ f.StructValue.Fields[field] = maskProtoValue(m, value)
}
return v
case *structpb.Value_ListValue:
@@ -628,7 +628,7 @@ func maskProtoValue(m Masker, value *structpb.Value) *structpb.Value {
//nolint
l := v.Kind.(*structpb.Value_ListValue)
for i, value := range l.ListValue.Values {
- kindValue.ListValue.Values[i] = maskProtoValue(m, value)
+ l.ListValue.Values[i] = maskProtoValue(m, value)
}
return v
}