Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
Fix semantic JSON masking for non-full maskers (#19540)
h3n4lb4ec1e6908a2
diff --git a/backend/api/v1/catalog_masking.go b/backend/api/v1/catalog_masking.goindex 0e6d936ca0..757bff7ef7 100644--- a/backend/api/v1/catalog_masking.go+++ b/backend/api/v1/catalog_masking.go@@ -1,6 +1,8 @@package v1import (+ "encoding/json"+"github.com/pkg/errors""github.com/bytebase/bytebase/backend/component/masker"@@ -110,7 +112,11 @@ func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*base.PathAST, oo, parentSemanticType = getObjectSchemaByPath(o, ast)if parentSemanticType != "" {if m, ok := semanticTypeToMasker[parentSemanticType]; ok {- result[key] = getJSONMemberFromRowValue(m.Mask(nil))+ maskedValue, err := applyMaskerToJSONMember(value, m)+ if err != nil {+ return nil, err+ }+ result[key] = maskedValuecontinue}}@@ -191,7 +197,7 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, semaif objectSchema.SemanticType != "" {// If the semantic type is found, replace the entire value directly.if m, ok := semanticTypeToMasker[objectSchema.SemanticType]; ok {- return getJSONMemberFromRowValue(m.Mask(nil)), nil+ return applyMaskerToJSONMember(data, m)}} else {// Otherwise, recursively walk the object.@@ -216,7 +222,7 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, semaif objectSchema.SemanticType != "" {// If the semantic type is found, replace the entire value directly.if m, ok := semanticTypeToMasker[objectSchema.SemanticType]; ok {- return getJSONMemberFromRowValue(m.Mask(nil)), nil+ return applyMaskerToJSONMember(data, m)}} else {arrayKind := objectSchema.GetArrayKind()@@ -252,6 +258,24 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, semareturn data, nil}+func applyMaskerToJSONMember(data any, m masker.Masker) (any, error) {+ if rowValue, ok := getRowValueFromJSONAtomicMember(data); ok {+ return getJSONMemberFromRowValue(m.Mask(&masker.MaskData{Data: rowValue})), nil+ }++ jsonBytes, err := json.Marshal(data)+ if err != nil {+ return nil, errors.Wrap(err, "failed to marshal JSON member")+ }+ return getJSONMemberFromRowValue(m.Mask(&masker.MaskData{+ Data: &v1pb.RowValue{+ Kind: &v1pb.RowValue_StringValue{+ StringValue: string(jsonBytes),+ },+ },+ })), nil+}+func applyMaskerToData(data any, m masker.Masker) (any, error) {switch data := data.(type) {case map[string]any:@@ -285,7 +309,8 @@ func applyMaskerToData(data any, m masker.Masker) (any, error) {func getJSONMemberFromRowValue(rowValue *v1pb.RowValue) any {switch rowValue := rowValue.Kind.(type) {- // TODO: Handle NULL, VALUE_VALUE, TIMESTAMP_VALUE, TIMESTAMPTZVALUE.+ case *v1pb.RowValue_NullValue:+ return nilcase *v1pb.RowValue_BoolValue:return rowValue.BoolValuecase *v1pb.RowValue_BytesValue:@@ -327,6 +352,5 @@ func getRowValueFromJSONAtomicMember(data any) (result *v1pb.RowValue, ok bool)Kind: &v1pb.RowValue_BoolValue{BoolValue: data},}, true}- // TODO: Handle NULL.return nil, false}diff --git a/backend/api/v1/catalog_masking_test.go b/backend/api/v1/catalog_masking_test.goindex 200e176c00..d04d91df0f 100644--- a/backend/api/v1/catalog_masking_test.go+++ b/backend/api/v1/catalog_masking_test.go@@ -135,6 +135,9 @@ func TestGetFirstSemanticTypeInPath(t *testing.T) {}func TestWalkAndMaskJSON(t *testing.T) {+ innerMasker, err := masker.NewInnerOuterMasker(storepb.Algorithm_InnerOuterMask_INNER, 1, 1, "*")+ require.NoError(t, err)+type testCase struct {description stringinput string@@ -319,6 +322,63 @@ func TestWalkAndMaskJSON(t *testing.T) {},want: `{"name": "John", "tags": "******"}`,},+ {+ description: "tagged field with object value is masked as opaque JSON string for non-full masker",+ input: `{"name": "John", "address": {"street": "123 Main St", "city": "NYC"}}`,+ objectSchema: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "address": {+ Type: storepb.ObjectSchema_OBJECT,+ SemanticType: "PII",+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "street": {Type: storepb.ObjectSchema_STRING},+ "city": {Type: storepb.ObjectSchema_STRING},+ },+ },+ },+ },+ },+ },+ },+ },+ semanticTypeToMasker: map[string]masker.Masker{+ "PII": innerMasker,+ },+ want: `{"name": "John", "address": "{***********************************}"}`,+ },+ {+ description: "tagged field with array value is masked as opaque JSON string for non-full masker",+ input: `{"name": "John", "tags": ["tag1", "tag2"]}`,+ objectSchema: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "tags": {+ Type: storepb.ObjectSchema_ARRAY,+ SemanticType: "PII",+ Kind: &storepb.ObjectSchema_ArrayKind_{+ ArrayKind: &storepb.ObjectSchema_ArrayKind{+ Kind: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_STRING,+ },+ },+ },+ },+ },+ },+ },+ },+ semanticTypeToMasker: map[string]masker.Masker{+ "PII": innerMasker,+ },+ want: `{"name": "John", "tags": "[*************]"}`,+ },{description: "mask following the field paths",input: `{@@ -406,3 +466,65 @@ func TestWalkAndMaskJSON(t *testing.T) {require.JSONEqf(t, tc.want, string(output), tc.description)}}++func TestWalkAndMaskJSONRecursiveSemanticTypeMasksJSONContainerAsString(t *testing.T) {+ innerMasker, err := masker.NewInnerOuterMasker(storepb.Algorithm_InnerOuterMask_INNER, 1, 1, "*")+ require.NoError(t, err)++ testCases := []struct {+ description string+ input string+ objectSchema *storepb.ObjectSchema+ want string+ }{+ {+ description: "root object semantic type is masked as opaque JSON string",+ input: `{"street": "123 Main St", "city": "NYC"}`,+ objectSchema: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ SemanticType: "PII",+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "street": {Type: storepb.ObjectSchema_STRING},+ "city": {Type: storepb.ObjectSchema_STRING},+ },+ },+ },+ },+ want: `"{***********************************}"`,+ },+ {+ description: "root array semantic type is masked as opaque JSON string",+ input: `["tag1", "tag2"]`,+ objectSchema: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_ARRAY,+ SemanticType: "PII",+ Kind: &storepb.ObjectSchema_ArrayKind_{+ ArrayKind: &storepb.ObjectSchema_ArrayKind{+ Kind: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_STRING,+ },+ },+ },+ },+ want: `"[*************]"`,+ },+ }++ for _, tc := range testCases {+ var input any+ err := json.Unmarshal([]byte(tc.input), &input)+ require.NoError(t, err, tc.description)++ got, err := walkAndMaskJSONRecursive(input, tc.objectSchema, map[string]masker.Masker{+ "PII": innerMasker,+ })+ require.NoError(t, err, tc.description)++ output, err := json.Marshal(got)+ require.NoError(t, err, tc.description)++ require.JSONEqf(t, tc.want, string(output), tc.description)+ }+}diff --git a/backend/component/masker/masker.go b/backend/component/masker/masker.goindex bacc6fd11b..162939c4db 100644--- a/backend/component/masker/masker.go+++ b/backend/component/masker/masker.go@@ -620,7 +620,7 @@ func maskProtoValue(m Masker, value *structpb.Value) *structpb.Value {//nolintf := v.Kind.(*structpb.Value_StructValue)for field, value := range f.StructValue.Fields {- kindValue.StructValue.Fields[field] = maskProtoValue(m, value)+ f.StructValue.Fields[field] = maskProtoValue(m, value)}return vcase *structpb.Value_ListValue:@@ -628,7 +628,7 @@ func maskProtoValue(m Masker, value *structpb.Value) *structpb.Value {//nolintl := v.Kind.(*structpb.Value_ListValue)for i, value := range l.ListValue.Values {- kindValue.ListValue.Values[i] = maskProtoValue(m, value)+ l.ListValue.Values[i] = maskProtoValue(m, value)}return v}