Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: implement data masking for Elasticsearch/OpenSearch queries (#19394)

h3n4l41921b809557
diff --git a/backend/api/v1/catalog_masking.go b/backend/api/v1/catalog_masking.go
index c75ab7e79b..e92eb0d559 100644
--- a/backend/api/v1/catalog_masking.go
+++ b/backend/api/v1/catalog_masking.go
@@ -110,11 +110,7 @@ func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*base.PathAST, o
o, parentSemanticType = getObjectSchemaByPath(o, ast)
if parentSemanticType != "" {
if m, ok := semanticTypeToMasker[parentSemanticType]; ok {
- maskedData, err := applyMaskerToData(value, m)
- if err != nil {
- return nil, err
- }
- result[key] = maskedData
+ result[key] = getJSONMemberFromRowValue(m.Mask(nil))
continue
}
}
@@ -182,13 +178,9 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, sema
switch data := data.(type) {
case map[string]any:
if objectSchema.SemanticType != "" {
- // If the outer semantic type is found, apply the masker recursively to the object.
+ // If the semantic type is found, replace the entire value directly.
if m, ok := semanticTypeToMasker[objectSchema.SemanticType]; ok {
- maskedData, err := applyMaskerToData(data, m)
- if err != nil {
- return nil, err
- }
- return maskedData, nil
+ return getJSONMemberFromRowValue(m.Mask(nil)), nil
}
} else {
// Otherwise, recursively walk the object.
@@ -211,13 +203,9 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, sema
return data, nil
case []any:
if objectSchema.SemanticType != "" {
- // If the outer semantic type is found, apply the masker recursively to the array.
+ // If the semantic type is found, replace the entire value directly.
if m, ok := semanticTypeToMasker[objectSchema.SemanticType]; ok {
- maskedData, err := applyMaskerToData(data, m)
- if err != nil {
- return nil, err
- }
- return maskedData, nil
+ return getJSONMemberFromRowValue(m.Mask(nil)), nil
}
} else {
arrayKind := objectSchema.GetArrayKind()
diff --git a/backend/api/v1/catalog_masking_elasticsearch.go b/backend/api/v1/catalog_masking_elasticsearch.go
new file mode 100644
index 0000000000..bbc49f56d4
--- /dev/null
+++ b/backend/api/v1/catalog_masking_elasticsearch.go
@@ -0,0 +1,484 @@
+package v1
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/component/masker"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+// lookupSemanticTypeByDotPath splits a dot-delimited field path and walks the ObjectSchema
+// to find the semantic type at the leaf.
+func lookupSemanticTypeByDotPath(dotPath string, objectSchema *storepb.ObjectSchema) string {
+ parts := strings.Split(dotPath, ".")
+ current := objectSchema
+ for _, part := range parts {
+ if current == nil {
+ return ""
+ }
+ if current.SemanticType != "" {
+ return current.SemanticType
+ }
+ structKind := current.GetStructKind()
+ if structKind == nil {
+ return ""
+ }
+ props := structKind.GetProperties()
+ if props == nil {
+ return ""
+ }
+ child, ok := props[part]
+ if !ok {
+ return ""
+ }
+ current = child
+ }
+ if current != nil {
+ return current.SemanticType
+ }
+ return ""
+}
+
+// maskElasticsearchSourceObject masks a single _source JSON object using
+// the ObjectSchema and semantic type maskers. It delegates to walkAndMaskJSONRecursive
+// from catalog_masking.go: when a field has a semantic type, the entire value is
+// replaced directly regardless of whether it is a primitive, object, or array.
+func maskElasticsearchSourceObject(source map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {
+ if objectSchema == nil {
+ return source, nil
+ }
+ masked, err := walkAndMaskJSONRecursive(source, objectSchema, semanticTypeToMasker)
+ if err != nil {
+ return nil, err
+ }
+ result, ok := masked.(map[string]any)
+ if !ok {
+ return source, nil
+ }
+ return result, nil
+}
+
+// maskElasticsearchHitFields masks the "fields" section of a hit.
+// Fields use dot-path keys with array values.
+func maskElasticsearchHitFields(hitMap map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {
+ fieldsVal, ok := hitMap["fields"]
+ if !ok {
+ return nil
+ }
+ fieldsMap, ok := fieldsVal.(map[string]any)
+ if !ok {
+ return nil
+ }
+ for dotPath, values := range fieldsMap {
+ semanticType := lookupSemanticTypeByDotPath(dotPath, objectSchema)
+ if semanticType == "" {
+ continue
+ }
+ m, ok := semanticTypeToMasker[semanticType]
+ if !ok {
+ continue
+ }
+ arr, ok := values.([]any)
+ if !ok {
+ continue
+ }
+ for i, val := range arr {
+ masked, err := applyMaskerToData(val, m)
+ if err != nil {
+ return err
+ }
+ arr[i] = masked
+ }
+ fieldsMap[dotPath] = arr
+ }
+ hitMap["fields"] = fieldsMap
+ return nil
+}
+
+// maskElasticsearchHitHighlight masks the "highlight" section of a hit.
+// Highlight uses dot-path keys with array values (HTML fragment strings).
+func maskElasticsearchHitHighlight(hitMap map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {
+ highlightVal, ok := hitMap["highlight"]
+ if !ok {
+ return nil
+ }
+ highlightMap, ok := highlightVal.(map[string]any)
+ if !ok {
+ return nil
+ }
+ for dotPath, values := range highlightMap {
+ semanticType := lookupSemanticTypeByDotPath(dotPath, objectSchema)
+ if semanticType == "" {
+ continue
+ }
+ m, ok := semanticTypeToMasker[semanticType]
+ if !ok {
+ continue
+ }
+ arr, ok := values.([]any)
+ if !ok {
+ continue
+ }
+ for i, val := range arr {
+ masked, err := applyMaskerToData(val, m)
+ if err != nil {
+ return err
+ }
+ arr[i] = masked
+ }
+ highlightMap[dotPath] = arr
+ }
+ hitMap["highlight"] = highlightMap
+ return nil
+}
+
+// maskElasticsearchHitSort masks the "sort" array of a hit based on the sort fields from the request.
+// sortFields is the ordered list of field names used in the sort clause (from request analysis).
+func maskElasticsearchHitSort(hitMap map[string]any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {
+ sortVal, ok := hitMap["sort"]
+ if !ok {
+ return nil
+ }
+ sortArr, ok := sortVal.([]any)
+ if !ok {
+ return nil
+ }
+ for i, val := range sortArr {
+ if i >= len(sortFields) {
+ break
+ }
+ fieldName := sortFields[i]
+ if fieldName == "" {
+ continue
+ }
+ semanticType := lookupSemanticTypeByDotPath(fieldName, objectSchema)
+ if semanticType == "" {
+ continue
+ }
+ m, ok := semanticTypeToMasker[semanticType]
+ if !ok {
+ continue
+ }
+ masked, err := applyMaskerToData(val, m)
+ if err != nil {
+ return err
+ }
+ sortArr[i] = masked
+ }
+ hitMap["sort"] = sortArr
+ return nil
+}
+
+// maskElasticsearchSingleHit masks _source, fields, highlight, sort, and inner_hits for a single hit.
+func maskElasticsearchSingleHit(hitMap map[string]any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {
+ if source, ok := hitMap["_source"].(map[string]any); ok {
+ masked, err := maskElasticsearchSourceObject(source, objectSchema, semanticTypeToMasker)
+ if err != nil {
+ return err
+ }
+ hitMap["_source"] = masked
+ }
+ if err := maskElasticsearchHitFields(hitMap, objectSchema, semanticTypeToMasker); err != nil {
+ return err
+ }
+ if err := maskElasticsearchHitHighlight(hitMap, objectSchema, semanticTypeToMasker); err != nil {
+ return err
+ }
+ if err := maskElasticsearchHitSort(hitMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {
+ return err
+ }
+ return maskElasticsearchHitInnerHits(hitMap, sortFields, objectSchema, semanticTypeToMasker)
+}
+
+// maskElasticsearchHitInnerHits masks the inner_hits section of a hit.
+// inner_hits.<name>.hits.hits[] has the same structure as outer hits,
+// so masking is applied recursively using the same ObjectSchema.
+func maskElasticsearchHitInnerHits(hitMap map[string]any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {
+ innerHitsVal, ok := hitMap["inner_hits"]
+ if !ok {
+ return nil
+ }
+ innerHitsMap, ok := innerHitsVal.(map[string]any)
+ if !ok {
+ return nil
+ }
+ for name, ihVal := range innerHitsMap {
+ if err := maskElasticsearchInnerHitGroup(name, ihVal, sortFields, objectSchema, semanticTypeToMasker); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// maskElasticsearchInnerHitGroup masks a single named inner_hits group (e.g. inner_hits.comments).
+func maskElasticsearchInnerHitGroup(name string, ihVal any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {
+ ihObj, ok := ihVal.(map[string]any)
+ if !ok {
+ return nil
+ }
+ hitsVal, ok := ihObj["hits"]
+ if !ok {
+ return nil
+ }
+ hitsObj, ok := hitsVal.(map[string]any)
+ if !ok {
+ return nil
+ }
+ hitsArray, ok := hitsObj["hits"].([]any)
+ if !ok {
+ return nil
+ }
+ for i, innerHit := range hitsArray {
+ innerHitMap, ok := innerHit.(map[string]any)
+ if !ok {
+ continue
+ }
+ if err := maskElasticsearchSingleHit(innerHitMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {
+ return errors.Wrapf(err, "failed to mask inner_hits.%s hit %d", name, i)
+ }
+ hitsArray[i] = innerHitMap
+ }
+ return nil
+}
+
+// maskElasticsearchHitsColumn masks _source, fields, highlight, sort, and inner_hits in each hit
+// within a hits column value. The hitsColumnJSON is the JSON string from the "hits" column,
+// e.g. {"total":{"value":1},"hits":[{"_source":{...},"fields":{...},"highlight":{...},"sort":[...]}]}.
+// sortFields is the ordered list of field names from the request's sort clause.
+func maskElasticsearchHitsColumn(hitsColumnJSON string, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {
+ var hitsObj map[string]any
+ if err := json.Unmarshal([]byte(hitsColumnJSON), &hitsObj); err != nil {
+ return "", errors.Wrap(err, "failed to unmarshal hits column")
+ }
+
+ hitsArray, ok := hitsObj["hits"].([]any)
+ if !ok {
+ return hitsColumnJSON, nil
+ }
+
+ for i, hit := range hitsArray {
+ hitMap, ok := hit.(map[string]any)
+ if !ok {
+ continue
+ }
+ if err := maskElasticsearchSingleHit(hitMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {
+ return "", errors.Wrapf(err, "failed to mask hit %d", i)
+ }
+ hitsArray[i] = hitMap
+ }
+ hitsObj["hits"] = hitsArray
+
+ out, err := json.Marshal(hitsObj)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to marshal masked hits column")
+ }
+ return string(out), nil
+}
+
+// maskElasticsearchDocSource masks the _source column value for a GET _doc response.
+// The sourceColumnJSON is the JSON string from the "_source" column, which is the raw document.
+func maskElasticsearchDocSource(sourceColumnJSON string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {
+ var source map[string]any
+ if err := json.Unmarshal([]byte(sourceColumnJSON), &source); err != nil {
+ return "", errors.Wrap(err, "failed to unmarshal _source column")
+ }
+
+ masked, err := maskElasticsearchSourceObject(source, objectSchema, semanticTypeToMasker)
+ if err != nil {
+ return "", err
+ }
+
+ out, err := json.Marshal(masked)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to marshal masked _source column")
+ }
+ return string(out), nil
+}
+
+// maskElasticsearchGetSourceColumn masks a single field value from a GET _source response.
+// In _source responses, each top-level document field becomes a separate column.
+// fieldName is the column name (= field name), valueJSON is the marshaled value.
+func maskElasticsearchGetSourceColumn(fieldName, valueJSON string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {
+ var childSchema *storepb.ObjectSchema
+ if structKind := objectSchema.GetStructKind(); structKind != nil {
+ if props := structKind.GetProperties(); props != nil {
+ childSchema = props[fieldName]
+ }
+ }
+ if childSchema == nil {
+ return valueJSON, nil
+ }
+
+ var value any
+ if err := json.Unmarshal([]byte(valueJSON), &value); err != nil {
+ return "", errors.Wrapf(err, "failed to unmarshal field %q", fieldName)
+ }
+
+ masked, err := walkAndMaskJSONRecursive(value, childSchema, semanticTypeToMasker)
+ if err != nil {
+ return "", err
+ }
+
+ out, err := json.Marshal(masked)
+ if err != nil {
+ return "", errors.Wrapf(err, "failed to marshal masked field %q", fieldName)
+ }
+ return string(out), nil
+}
+
+// maskElasticsearchMGetSource masks _source in each doc within a _mget response.
+// The docsColumnJSON is the JSON string from the "docs" column.
+func maskElasticsearchMGetSource(docsColumnJSON string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {
+ var docs []any
+ if err := json.Unmarshal([]byte(docsColumnJSON), &docs); err != nil {
+ return "", errors.Wrap(err, "failed to unmarshal docs column")
+ }
+
+ for i, doc := range docs {
+ docMap, ok := doc.(map[string]any)
+ if !ok {
+ continue
+ }
+ if source, ok := docMap["_source"].(map[string]any); ok {
+ masked, err := maskElasticsearchSourceObject(source, objectSchema, semanticTypeToMasker)
+ if err != nil {
+ return "", errors.Wrapf(err, "failed to mask _source in doc %d", i)
+ }
+ docMap["_source"] = masked
+ }
+ docs[i] = docMap
+ }
+
+ out, err := json.Marshal(docs)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to marshal masked docs column")
+ }
+ return string(out), nil
+}
+
+// maskElasticsearchMSearchResponses masks the "responses" column from a _msearch response.
+// Each element in the responses array is a full search response with hits.
+func maskElasticsearchMSearchResponses(responsesColumnJSON string, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {
+ var responses []any
+ if err := json.Unmarshal([]byte(responsesColumnJSON), &responses); err != nil {
+ return "", errors.Wrap(err, "failed to unmarshal responses column")
+ }
+
+ for i, resp := range responses {
+ respMap, ok := resp.(map[string]any)
+ if !ok {
+ continue
+ }
+ if err := maskElasticsearchMSearchSingleResponse(respMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {
+ return "", errors.Wrapf(err, "failed to mask response %d", i)
+ }
+ }
+
+ out, err := json.Marshal(responses)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to marshal masked responses column")
+ }
+ return string(out), nil
+}
+
+// maskElasticsearchMSearchSingleResponse masks the hits in a single _msearch response element.
+func maskElasticsearchMSearchSingleResponse(respMap map[string]any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {
+ hitsVal, ok := respMap["hits"]
+ if !ok {
+ return nil
+ }
+ hitsObj, ok := hitsVal.(map[string]any)
+ if !ok {
+ return nil
+ }
+ hitsArray, ok := hitsObj["hits"].([]any)
+ if !ok {
+ return nil
+ }
+ for j, hit := range hitsArray {
+ hitMap, ok := hit.(map[string]any)
+ if !ok {
+ continue
+ }
+ if err := maskElasticsearchSingleHit(hitMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {
+ return errors.Wrapf(err, "failed to mask hit %d", j)
+ }
+ hitsArray[j] = hitMap
+ }
+ return nil
+}
+
+// checkElasticsearchRequestBlocked returns an error if the request must be blocked
+// because masking policies exist on the target index.
+func checkElasticsearchRequestBlocked(analysis *esparser.RequestAnalysis) error {
+ if analysis.API == esparser.APIBlocked {
+ return errors.New("this Elasticsearch API is not supported when data masking is configured on the target index")
+ }
+ if len(analysis.BlockedFeatures) > 0 {
+ names := make([]string, 0, len(analysis.BlockedFeatures))
+ for _, f := range analysis.BlockedFeatures {
+ names = append(names, esparser.BlockedFeatureNames[f])
+ }
+ return errors.Errorf("the following features are not supported when data masking is configured: %s", strings.Join(names, ", "))
+ }
+ return nil
+}
+
+// getElasticsearchIndexObjectSchema retrieves the ObjectSchema for an ES index from the database config.
+func getElasticsearchIndexObjectSchema(ctx context.Context, stores *store.Store, instanceID string, databaseName string, indexName string) (*storepb.ObjectSchema, error) {
+ dbMetadata, err := stores.GetDBSchema(ctx, &store.FindDBSchemaMessage{
+ InstanceID: instanceID,
+ DatabaseName: databaseName,
+ })
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to get database schema: %q", databaseName)
+ }
+
+ if dbMetadata == nil {
+ return nil, nil
+ }
+
+ schemas := dbMetadata.GetConfig().GetSchemas()
+ if len(schemas) == 0 {
+ return nil, nil
+ }
+
+ schema := schemas[0]
+ tables := schema.GetTables()
+ for _, table := range tables {
+ if table.GetName() == indexName {
+ return table.GetObjectSchema(), nil
+ }
+ }
+
+ return nil, nil
+}
+
+// objectSchemaHasSemanticTypes recursively checks if any node in the ObjectSchema has a semantic type set.
+func objectSchemaHasSemanticTypes(os *storepb.ObjectSchema) bool {
+ if os == nil {
+ return false
+ }
+ if os.SemanticType != "" {
+ return true
+ }
+ if sk := os.GetStructKind(); sk != nil {
+ for _, prop := range sk.GetProperties() {
+ if objectSchemaHasSemanticTypes(prop) {
+ return true
+ }
+ }
+ }
+ if ak := os.GetArrayKind(); ak != nil {
+ if objectSchemaHasSemanticTypes(ak.GetKind()) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/backend/api/v1/catalog_masking_elasticsearch_test.go b/backend/api/v1/catalog_masking_elasticsearch_test.go
new file mode 100644
index 0000000000..48ed67e6b8
--- /dev/null
+++ b/backend/api/v1/catalog_masking_elasticsearch_test.go
@@ -0,0 +1,314 @@
+package v1
+
+import (
+ "encoding/json"
+ "io"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "gopkg.in/yaml.v3"
+
+ "github.com/bytebase/bytebase/backend/component/masker"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"
+)
+
+// testElasticsearchObjectSchema creates a schema where "email" and "contact.phone" are masked.
+func testElasticsearchObjectSchema() *storepb.ObjectSchema {
+ return &storepb.ObjectSchema{
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "name": {Type: storepb.ObjectSchema_STRING},
+ "email": {Type: storepb.ObjectSchema_STRING, SemanticType: "bb.default"},
+ "age": {Type: storepb.ObjectSchema_NUMBER},
+ "contact": {
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "phone": {Type: storepb.ObjectSchema_STRING, SemanticType: "bb.default"},
+ "city": {Type: storepb.ObjectSchema_STRING},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+func testElasticsearchMaskerMap() map[string]masker.Masker {
+ return map[string]masker.Masker{
+ "bb.default": masker.NewDefaultFullMasker(),
+ }
+}
+
+type elasticsearchTestData struct {
+ LookupSemanticType []lookupSemanticTypeCase `yaml:"lookupSemanticType"`
+ MaskHitsColumn []maskJSONCase `yaml:"maskHitsColumn"`
+ MaskDocSource []maskJSONCase `yaml:"maskDocSource"`
+ MaskGetSourceColumn []maskGetSourceCase `yaml:"maskGetSourceColumn"`
+ MaskMGetSource []maskJSONCase `yaml:"maskMGetSource"`
+ MaskMSearchResponses []maskJSONCase `yaml:"maskMSearchResponses"`
+ MaskInnerHits []maskJSONCase `yaml:"maskInnerHits"`
+ CheckBlocked []checkBlockedCase `yaml:"checkBlocked"`
+}
+
+type lookupSemanticTypeCase struct {
+ Description string `yaml:"description"`
+ DotPath string `yaml:"dotPath"`
+ WantType string `yaml:"wantType"`
+}
+
+type maskJSONCase struct {
+ Description string `yaml:"description"`
+ Input string `yaml:"input"`
+ Want string `yaml:"want"`
+ SortFields []string `yaml:"sortFields"`
+}
+
+type maskGetSourceCase struct {
+ Description string `yaml:"description"`
+ FieldName string `yaml:"fieldName"`
+ Input string `yaml:"input"`
+ Want string `yaml:"want"`
+}
+
+type checkBlockedCase struct {
+ Description string `yaml:"description"`
+ API esparser.MaskableAPI `yaml:"api"`
+ Index string `yaml:"index"`
+ BlockedFeatures []esparser.BlockedFeature `yaml:"blockedFeatures"`
+ WantError bool `yaml:"wantError"`
+ WantErrorContains string `yaml:"wantErrorContains"`
+}
+
+func loadElasticsearchTestData(t *testing.T) *elasticsearchTestData {
+ t.Helper()
+ f, err := os.Open("test-data/elasticsearch_masking.yaml")
+ require.NoError(t, err)
+ raw, err := io.ReadAll(f)
+ require.NoError(t, err)
+ require.NoError(t, f.Close())
+
+ var data elasticsearchTestData
+ require.NoError(t, yaml.Unmarshal(raw, &data))
+ return &data
+}
+
+// requireJSONEqual compares two JSON strings by unmarshalling and comparing as Go values.
+func requireJSONEqual(t *testing.T, want, got string) {
+ t.Helper()
+ var wantVal, gotVal any
+ require.NoError(t, json.Unmarshal([]byte(want), &wantVal))
+ require.NoError(t, json.Unmarshal([]byte(got), &gotVal))
+ require.Equal(t, wantVal, gotVal)
+}
+
+func TestLookupSemanticTypeByDotPath(t *testing.T) {
+ td := loadElasticsearchTestData(t)
+ schema := testElasticsearchObjectSchema()
+
+ for _, tc := range td.LookupSemanticType {
+ t.Run(tc.Description, func(t *testing.T) {
+ got := lookupSemanticTypeByDotPath(tc.DotPath, schema)
+ require.Equal(t, tc.WantType, got)
+ })
+ }
+
+ t.Run("nil schema", func(t *testing.T) {
+ got := lookupSemanticTypeByDotPath("email", nil)
+ require.Equal(t, "", got)
+ })
+}
+
+func TestMaskElasticsearchHitsColumn(t *testing.T) {
+ td := loadElasticsearchTestData(t)
+ schema := testElasticsearchObjectSchema()
+ maskers := testElasticsearchMaskerMap()
+
+ for _, tc := range td.MaskHitsColumn {
+ t.Run(tc.Description, func(t *testing.T) {
+ result, err := maskElasticsearchHitsColumn(tc.Input, tc.SortFields, schema, maskers)
+ require.NoError(t, err)
+ requireJSONEqual(t, tc.Want, result)
+ })
+ }
+}
+
+func TestMaskElasticsearchDocSource(t *testing.T) {
+ td := loadElasticsearchTestData(t)
+ schema := testElasticsearchObjectSchema()
+ maskers := testElasticsearchMaskerMap()
+
+ for _, tc := range td.MaskDocSource {
+ t.Run(tc.Description, func(t *testing.T) {
+ result, err := maskElasticsearchDocSource(tc.Input, schema, maskers)
+ require.NoError(t, err)
+ requireJSONEqual(t, tc.Want, result)
+ })
+ }
+}
+
+func TestMaskElasticsearchGetSourceColumn(t *testing.T) {
+ td := loadElasticsearchTestData(t)
+ schema := testElasticsearchObjectSchema()
+ maskers := testElasticsearchMaskerMap()
+
+ for _, tc := range td.MaskGetSourceColumn {
+ t.Run(tc.Description, func(t *testing.T) {
+ result, err := maskElasticsearchGetSourceColumn(tc.FieldName, tc.Input, schema, maskers)
+ require.NoError(t, err)
+ requireJSONEqual(t, tc.Want, result)
+ })
+ }
+}
+
+func TestMaskElasticsearchMGetSource(t *testing.T) {
+ td := loadElasticsearchTestData(t)
+ schema := testElasticsearchObjectSchema()
+ maskers := testElasticsearchMaskerMap()
+
+ for _, tc := range td.MaskMGetSource {
+ t.Run(tc.Description, func(t *testing.T) {
+ result, err := maskElasticsearchMGetSource(tc.Input, schema, maskers)
+ require.NoError(t, err)
+ requireJSONEqual(t, tc.Want, result)
+ })
+ }
+}
+
+func TestMaskElasticsearchInnerHits(t *testing.T) {
+ td := loadElasticsearchTestData(t)
+ schema := testElasticsearchObjectSchema()
+ maskers := testElasticsearchMaskerMap()
+
+ for _, tc := range td.MaskInnerHits {
+ t.Run(tc.Description, func(t *testing.T) {
+ result, err := maskElasticsearchHitsColumn(tc.Input, tc.SortFields, schema, maskers)
+ require.NoError(t, err)
+ requireJSONEqual(t, tc.Want, result)
+ })
+ }
+}
+
+func TestMaskElasticsearchMSearchResponses(t *testing.T) {
+ td := loadElasticsearchTestData(t)
+ schema := testElasticsearchObjectSchema()
+ maskers := testElasticsearchMaskerMap()
+
+ for _, tc := range td.MaskMSearchResponses {
+ t.Run(tc.Description, func(t *testing.T) {
+ result, err := maskElasticsearchMSearchResponses(tc.Input, tc.SortFields, schema, maskers)
+ require.NoError(t, err)
+ requireJSONEqual(t, tc.Want, result)
+ })
+ }
+}
+
+func TestMaskElasticsearchSourceObjectDirectReplacement(t *testing.T) {
+ maskers := map[string]masker.Masker{
+ "bb.default": masker.NewDefaultFullMasker(),
+ }
+
+ tests := []struct {
+ description string
+ input map[string]any
+ schema *storepb.ObjectSchema
+ want map[string]any
+ }{
+ {
+ description: "tagged field with nested object is replaced directly",
+ input: map[string]any{
+ "name": "Alice",
+ "contact": map[string]any{
+ "phone": "555-1234",
+ "city": "NYC",
+ },
+ },
+ schema: &storepb.ObjectSchema{
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "name": {Type: storepb.ObjectSchema_STRING},
+ "contact": {
+ Type: storepb.ObjectSchema_OBJECT,
+ SemanticType: "bb.default",
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "phone": {Type: storepb.ObjectSchema_STRING},
+ "city": {Type: storepb.ObjectSchema_STRING},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: map[string]any{
+ "name": "Alice",
+ "contact": "******",
+ },
+ },
+ {
+ description: "tagged field with array value is replaced directly",
+ input: map[string]any{
+ "name": "Alice",
+ "tags": []any{"tag1", "tag2"},
+ },
+ schema: &storepb.ObjectSchema{
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "name": {Type: storepb.ObjectSchema_STRING},
+ "tags": {
+ Type: storepb.ObjectSchema_ARRAY,
+ SemanticType: "bb.default",
+ },
+ },
+ },
+ },
+ },
+ want: map[string]any{
+ "name": "Alice",
+ "tags": "******",
+ },
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.description, func(t *testing.T) {
+ got, err := maskElasticsearchSourceObject(tc.input, tc.schema, maskers)
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
+
+func TestCheckElasticsearchRequestBlocked(t *testing.T) {
+ td := loadElasticsearchTestData(t)
+
+ for _, tc := range td.CheckBlocked {
+ t.Run(tc.Description, func(t *testing.T) {
+ analysis := &esparser.RequestAnalysis{
+ API: tc.API,
+ Index: tc.Index,
+ BlockedFeatures: tc.BlockedFeatures,
+ }
+ err := checkElasticsearchRequestBlocked(analysis)
+ if tc.WantError {
+ require.Error(t, err)
+ if tc.WantErrorContains != "" {
+ require.Contains(t, err.Error(), tc.WantErrorContains)
+ }
+ } else {
+ require.NoError(t, err)
+ }
+ })
+ }
+}
diff --git a/backend/api/v1/catalog_masking_test.go b/backend/api/v1/catalog_masking_test.go
index dc2cd5bbd7..200e176c00 100644
--- a/backend/api/v1/catalog_masking_test.go
+++ b/backend/api/v1/catalog_masking_test.go
@@ -223,7 +223,7 @@ func TestWalkAndMaskJSON(t *testing.T) {
want: `{"name": "John", "information": {"ssn": "******"}}`,
},
{
- description: "Recursive apply the masker to the object",
+ description: "tagged root replaces each field value directly",
input: `{"name": "John", "information": {"ssn": "123-45-6789"}}`,
objectSchema: &storepb.ObjectSchema{
Type: storepb.ObjectSchema_OBJECT,
@@ -232,7 +232,7 @@ func TestWalkAndMaskJSON(t *testing.T) {
semanticTypeToMasker: map[string]masker.Masker{
"SSN": masker.NewFullMasker("******"),
},
- want: `{"name": "******", "information": {"ssn": "******"}}`,
+ want: `{"name": "******", "information": "******"}`,
},
{
description: "Mask the array",
@@ -262,6 +262,63 @@ func TestWalkAndMaskJSON(t *testing.T) {
},
want: `{"name": "John", "information": ["******", "******"]}`,
},
+ {
+ description: "tagged field with object value is replaced directly",
+ input: `{"name": "John", "address": {"street": "123 Main St", "city": "NYC"}}`,
+ objectSchema: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "address": {
+ Type: storepb.ObjectSchema_OBJECT,
+ SemanticType: "PII",
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "street": {Type: storepb.ObjectSchema_STRING},
+ "city": {Type: storepb.ObjectSchema_STRING},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ semanticTypeToMasker: map[string]masker.Masker{
+ "PII": masker.NewFullMasker("******"),
+ },
+ want: `{"name": "John", "address": "******"}`,
+ },
+ {
+ description: "tagged field with array value is replaced directly",
+ input: `{"name": "John", "tags": ["tag1", "tag2", "tag3"]}`,
+ objectSchema: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_OBJECT,
+ Kind: &storepb.ObjectSchema_StructKind_{
+ StructKind: &storepb.ObjectSchema_StructKind{
+ Properties: map[string]*storepb.ObjectSchema{
+ "tags": {
+ Type: storepb.ObjectSchema_ARRAY,
+ SemanticType: "PII",
+ Kind: &storepb.ObjectSchema_ArrayKind_{
+ ArrayKind: &storepb.ObjectSchema_ArrayKind{
+ Kind: &storepb.ObjectSchema{
+ Type: storepb.ObjectSchema_STRING,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ semanticTypeToMasker: map[string]masker.Masker{
+ "PII": masker.NewFullMasker("******"),
+ },
+ want: `{"name": "John", "tags": "******"}`,
+ },
{
description: "mask following the field paths",
input: `{
@@ -314,9 +371,7 @@ func TestWalkAndMaskJSON(t *testing.T) {
},
want: `{
"firstName": "John",
- "address": {
- "street": "******"
- },
+ "address": "******",
"creditCard": {
"number": "******"
}
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index 3da26a4ba5..5050b9fba7 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -32,6 +32,7 @@ import (
"github.com/bytebase/bytebase/backend/plugin/advisor/code"
"github.com/bytebase/bytebase/backend/plugin/db"
parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"
"github.com/bytebase/bytebase/backend/plugin/schema"
"github.com/bytebase/bytebase/backend/runner/schemasync"
"github.com/bytebase/bytebase/backend/store"
@@ -584,6 +585,45 @@ func queryRetry(
}
}
+ // Pre-execution check for Elasticsearch: reject blocked APIs and predicate violations
+ // before sending the query to the server.
+ if !queryContext.Explain && !queryContext.SkipMasking &&
+ instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH &&
+ licenseService.IsFeatureEnabledForInstance(v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil {
+ for _, stmt := range statements {
+ parsed, parseErr := esparser.ParseElasticsearchREST(stmt.Text)
+ if parseErr != nil || len(parsed.Requests) == 0 {
+ continue
+ }
+ req := parsed.Requests[0]
+ analysis := esparser.AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))
+ if analysis.API == esparser.APIUnsupported {
+ continue
+ }
+ indexName := analysis.Index
+ if indexName == "" {
+ continue
+ }
+ objectSchema, schemaErr := getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, indexName)
+ if schemaErr != nil {
+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.Wrapf(schemaErr, "failed to get object schema for index %q", indexName))
+ }
+ if objectSchema == nil || !objectSchemaHasSemanticTypes(objectSchema) {
+ continue
+ }
+ if err := checkElasticsearchRequestBlocked(analysis); err != nil {
+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)
+ }
+ for _, field := range analysis.PredicateFields {
+ semanticType := lookupSemanticTypeByDotPath(field, objectSchema)
+ if semanticType != "" {
+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,
+ errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", field, semanticType))
+ }
+ }
+ }
+ }
+
slog.Debug("start execute with timeout", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName), slog.String("statement", originalStatement))
results, duration, queryErr := executeWithTimeout(
ctx,
@@ -730,6 +770,90 @@ func queryRetry(
}
}
}
+ } else if instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH {
+ for _, result := range results {
+ parsed, err := esparser.ParseElasticsearchREST(result.Statement)
+ if err != nil || len(parsed.Requests) == 0 {
+ continue
+ }
+ req := parsed.Requests[0]
+ analysis := esparser.AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))
+
+ if analysis.API == esparser.APIUnsupported || analysis.API == esparser.APIBlocked {
+ continue
+ }
+
+ indexName := analysis.Index
+ if indexName == "" {
+ continue
+ }
+ objectSchema, err := getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, indexName)
+ if err != nil {
+ return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get object schema for index %q", indexName))
+ }
+ if objectSchema == nil || !objectSchemaHasSemanticTypes(objectSchema) {
+ continue
+ }
+
+ semanticTypeToMaskerMap, err := buildSemanticTypeToMaskerMap(ctx, stores)
+ if err != nil {
+ return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.New(err.Error()))
+ }
+
+ // Apply masking based on API type.
+ for _, row := range result.Rows {
+ for colIdx, colName := range result.ColumnNames {
+ if colIdx >= len(row.Values) {
+ break
+ }
+ value := row.Values[colIdx].GetStringValue()
+ if value == "" {
+ continue
+ }
+
+ var maskedValue string
+ var maskErr error
+
+ switch analysis.API {
+ case esparser.APIMaskSearch:
+ switch colName {
+ case "hits":
+ maskedValue, maskErr = maskElasticsearchHitsColumn(value, analysis.SortFields, objectSchema, semanticTypeToMaskerMap)
+ case "responses":
+ maskedValue, maskErr = maskElasticsearchMSearchResponses(value, analysis.SortFields, objectSchema, semanticTypeToMaskerMap)
+ default:
+ continue
+ }
+ case esparser.APIMaskGetDoc, esparser.APIMaskExplain:
+ if colName == "_source" {
+ maskedValue, maskErr = maskElasticsearchDocSource(value, objectSchema, semanticTypeToMaskerMap)
+ } else {
+ continue
+ }
+ case esparser.APIMaskGetSource:
+ maskedValue, maskErr = maskElasticsearchGetSourceColumn(colName, value, objectSchema, semanticTypeToMaskerMap)
+ case esparser.APIMaskMGet:
+ if colName == "docs" {
+ maskedValue, maskErr = maskElasticsearchMGetSource(value, objectSchema, semanticTypeToMaskerMap)
+ } else {
+ continue
+ }
+ default:
+ continue
+ }
+
+ if maskErr != nil {
+ return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.Wrapf(maskErr, "failed to mask ES response"))
+ }
+
+ row.Values[colIdx] = &v1pb.RowValue{
+ Kind: &v1pb.RowValue_StringValue{
+ StringValue: maskedValue,
+ },
+ }
+ }
+ }
+ }
} else {
masker := NewQueryResultMasker(stores)
if err := masker.MaskResults(ctx, spans, results, instance, user); err != nil {
diff --git a/backend/api/v1/test-data/elasticsearch_masking.yaml b/backend/api/v1/test-data/elasticsearch_masking.yaml
new file mode 100644
index 0000000000..d2dc92f946
--- /dev/null
+++ b/backend/api/v1/test-data/elasticsearch_masking.yaml
@@ -0,0 +1,169 @@
+# Test data for Elasticsearch response masking.
+#
+# ObjectSchema used by all tests:
+# - "email" : STRING, semanticType "bb.default"
+# - "contact.phone" : STRING, semanticType "bb.default"
+# - "name" : STRING (no semantic type)
+# - "age" : NUMBER (no semantic type)
+# - "contact.city" : STRING (no semantic type)
+#
+# The masker for "bb.default" is DefaultFullMasker which replaces values with "******".
+#
+# MaskableAPI enum: APIUnsupported=0, APIMaskSearch=1, APIMaskGetDoc=2, APIMaskGetSource=3, APIMaskMGet=4, APIMaskExplain=5, APIBlocked=6
+# BlockedFeature enum: BlockedFeatureAggs=0, BlockedFeatureSuggest=1, BlockedFeatureScriptFields=2,
+# BlockedFeatureRuntimeMappings=3, BlockedFeatureStoredFields=4, BlockedFeatureDocvalueFields=5
+
+lookupSemanticType:
+ - description: "email field"
+ dotPath: "email"
+ wantType: "bb.default"
+ - description: "nested contact.phone"
+ dotPath: "contact.phone"
+ wantType: "bb.default"
+ - description: "name field (no semantic type)"
+ dotPath: "name"
+ wantType: ""
+ - description: "age field (no semantic type)"
+ dotPath: "age"
+ wantType: ""
+ - description: "contact.city (no semantic type)"
+ dotPath: "contact.city"
+ wantType: ""
+ - description: "unknown field"
+ dotPath: "unknown"
+ wantType: ""
+ - description: "contact.unknown"
+ dotPath: "contact.unknown"
+ wantType: ""
+ - description: "deeply.nested.path"
+ dotPath: "deeply.nested.path"
+ wantType: ""
+ - description: "empty path"
+ dotPath: ""
+ wantType: ""
+
+maskHitsColumn:
+ - description: "masks _source fields"
+ input: '{"total":{"value":1},"hits":[{"_index":"test","_id":"1","_source":{"name":"Alice","email":"[email protected]","contact":{"phone":"555-1234","city":"NYC"}}}]}'
+ want: '{"total":{"value":1},"hits":[{"_index":"test","_id":"1","_source":{"name":"Alice","email":"******","contact":{"phone":"******","city":"NYC"}}}]}'
+ - description: "no hits array"
+ input: '{"total":{"value":0}}'
+ want: '{"total":{"value":0}}'
+ - description: "hit without _source"
+ input: '{"total":{"value":1},"hits":[{"_index":"test","_id":"1"}]}'
+ want: '{"total":{"value":1},"hits":[{"_index":"test","_id":"1"}]}'
+ - description: "multiple hits"
+ input: '{"total":{"value":2},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"}},{"_id":"2","_source":{"name":"Bob","email":"[email protected]"}}]}'
+ want: '{"total":{"value":2},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"}},{"_id":"2","_source":{"name":"Bob","email":"******"}}]}'
+ - description: "masks fields section"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"fields":{"email":["[email protected]"],"contact.phone":["555-1234"],"name":["Alice"]}}]}'
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"fields":{"email":["******"],"contact.phone":["******"],"name":["Alice"]}}]}'
+ - description: "masks highlight section"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"highlight":{"email":["<em>alice</em>@example.com"],"name":["<em>Alice</em>"]}}]}'
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"highlight":{"email":["******"],"name":["<em>Alice</em>"]}}]}'
+ - description: "masks sort array"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"sort":[1234567890,"[email protected]"]}]}'
+ sortFields:
+ - "age"
+ - "email"
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"sort":[1234567890,"******"]}]}'
+ - description: "masks fields highlight and sort together"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"fields":{"email":["[email protected]"],"contact.phone":["555-1234"],"name":["Alice"]},"highlight":{"email":["<em>alice</em>@example.com"],"name":["<em>Alice</em>"]},"sort":[1234567890,"[email protected]"]}]}'
+ sortFields:
+ - "age"
+ - "email"
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"fields":{"email":["******"],"contact.phone":["******"],"name":["Alice"]},"highlight":{"email":["******"],"name":["<em>Alice</em>"]},"sort":[1234567890,"******"]}]}'
+ - description: "sort with fewer sortFields than sort values"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","sort":[1234567890,"[email protected]","extra"]}]}'
+ sortFields:
+ - "age"
+ want: '{"total":{"value":1},"hits":[{"_id":"1","sort":[1234567890,"[email protected]","extra"]}]}'
+
+maskDocSource:
+ - description: "masks _source column"
+ input: '{"name":"Alice","email":"[email protected]","contact":{"phone":"555-1234","city":"NYC"}}'
+ want: '{"name":"Alice","email":"******","contact":{"phone":"******","city":"NYC"}}'
+
+maskGetSourceColumn:
+ - description: "masks sensitive field"
+ fieldName: "email"
+ input: '"[email protected]"'
+ want: '"******"'
+ - description: "does not mask non-sensitive field"
+ fieldName: "name"
+ input: '"Alice"'
+ want: '"Alice"'
+ - description: "masks nested object field"
+ fieldName: "contact"
+ input: '{"phone":"555-1234","city":"NYC"}'
+ want: '{"phone":"******","city":"NYC"}'
+ - description: "unknown field passes through"
+ fieldName: "unknown_field"
+ input: '"value"'
+ want: '"value"'
+
+maskMGetSource:
+ - description: "masks docs"
+ input: '[{"_index":"test","_id":"1","found":true,"_source":{"name":"Alice","email":"[email protected]"}},{"_index":"test","_id":"2","found":true,"_source":{"name":"Bob","email":"[email protected]"}}]'
+ want: '[{"_index":"test","_id":"1","found":true,"_source":{"name":"Alice","email":"******"}},{"_index":"test","_id":"2","found":true,"_source":{"name":"Bob","email":"******"}}]'
+
+maskMSearchResponses:
+ - description: "masks each response"
+ input: '[{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"}}]}},{"hits":{"total":{"value":1},"hits":[{"_id":"2","_source":{"name":"Bob","email":"[email protected]","contact":{"phone":"555-1234","city":"NYC"}}}]}}]'
+ want: '[{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"}}]}},{"hits":{"total":{"value":1},"hits":[{"_id":"2","_source":{"name":"Bob","email":"******","contact":{"phone":"******","city":"NYC"}}}]}}]'
+ - description: "empty responses array"
+ input: '[]'
+ want: '[]'
+ - description: "response without hits passes through"
+ input: '[{"status":200},{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"}}]}}]'
+ want: '[{"status":200},{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"}}]}}]'
+ - description: "response with inner_hits"
+ input: '[{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"inner_hits":{"comments":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Bob","email":"[email protected]"}}]}}}}]}}]'
+ want: '[{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"inner_hits":{"comments":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Bob","email":"******"}}]}}}}]}}]'
+
+maskInnerHits:
+ - description: "masks inner_hits _source"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"inner_hits":{"comments":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Bob","email":"[email protected]","contact":{"phone":"555-9999","city":"LA"}}}]}}}}]}'
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"inner_hits":{"comments":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Bob","email":"******","contact":{"phone":"******","city":"LA"}}}]}}}}]}'
+ - description: "masks inner_hits fields and highlight"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"inner_hits":{"nested":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Inner","email":"[email protected]"},"fields":{"email":["[email protected]"],"name":["Inner"]},"highlight":{"email":["<em>inner</em>@example.com"]}}]}}}}]}'
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"inner_hits":{"nested":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Inner","email":"******"},"fields":{"email":["******"],"name":["Inner"]},"highlight":{"email":["******"]}}]}}}}]}'
+ - description: "masks nested inner_hits recursively"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Root","email":"[email protected]"},"inner_hits":{"level1":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"L1","email":"[email protected]"},"inner_hits":{"level2":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"L2","email":"[email protected]"}}]}}}}]}}}}]}'
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Root","email":"******"},"inner_hits":{"level1":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"L1","email":"******"},"inner_hits":{"level2":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"L2","email":"******"}}]}}}}]}}}}]}'
+ - description: "no inner_hits passes through"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"}}]}'
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"}}]}'
+ - description: "multiple inner_hits groups"
+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"inner_hits":{"group_a":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"GA","email":"[email protected]"}}]}},"group_b":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"GB","email":"[email protected]"}}]}}}}]}'
+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"inner_hits":{"group_a":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"GA","email":"******"}}]}},"group_b":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"GB","email":"******"}}]}}}}]}'
+
+checkBlocked:
+ - description: "allowed search"
+ api: 1
+ index: "my-index"
+ wantError: false
+ - description: "blocked API"
+ api: 6
+ index: "my-index"
+ wantError: true
+ wantErrorContains: "not supported when data masking"
+ - description: "blocked feature aggs"
+ api: 1
+ index: "my-index"
+ blockedFeatures:
+ - 0
+ wantError: true
+ wantErrorContains: "aggregations"
+ - description: "multiple blocked features"
+ api: 1
+ index: "my-index"
+ blockedFeatures:
+ - 0
+ - 1
+ - 2
+ wantError: true
+ wantErrorContains: "aggregations"
+ - description: "unsupported API passes through"
+ api: 0
+ wantError: false
diff --git a/backend/plugin/parser/elasticsearch/masking.go b/backend/plugin/parser/elasticsearch/masking.go
new file mode 100644
index 0000000000..70d5fef3ea
--- /dev/null
+++ b/backend/plugin/parser/elasticsearch/masking.go
@@ -0,0 +1,493 @@
+package elasticsearch
+
+import (
+ "encoding/json"
+ "slices"
+ "strings"
+)
+
+// MaskableAPI categorizes ES endpoints for masking decisions.
+type MaskableAPI int
+
+const (
+ // APIUnsupported means the endpoint is not relevant to masking (no user data).
+ APIUnsupported MaskableAPI = iota
+ // APIMaskSearch covers _search and _msearch.
+ APIMaskSearch
+ // APIMaskGetDoc covers GET /<index>/_doc/<id>.
+ APIMaskGetDoc
+ // APIMaskGetSource covers GET /<index>/_source/<id>.
+ APIMaskGetSource
+ // APIMaskMGet covers _mget.
+ APIMaskMGet
+ // APIMaskExplain covers _explain.
+ APIMaskExplain
+ // APIBlocked means the endpoint must be rejected when masking is active.
+ APIBlocked
+)
+
+// BlockedFeature identifies a feature in the request body that must be blocked.
+type BlockedFeature int
+
+const (
+ BlockedFeatureAggs BlockedFeature = iota
+ BlockedFeatureSuggest
+ BlockedFeatureScriptFields
+ BlockedFeatureRuntimeMappings
+ BlockedFeatureStoredFields
+ BlockedFeatureDocvalueFields
+)
+
+// BlockedFeatureNames maps BlockedFeature to human-readable names for error messages.
+var BlockedFeatureNames = map[BlockedFeature]string{
+ BlockedFeatureAggs: "aggregations",
+ BlockedFeatureSuggest: "suggest",
+ BlockedFeatureScriptFields: "script_fields",
+ BlockedFeatureRuntimeMappings: "runtime_mappings",
+ BlockedFeatureStoredFields: "stored_fields",
+ BlockedFeatureDocvalueFields: "docvalue_fields",
+}
+
+// RequestAnalysis is the result of analyzing an ES REST request for masking.
+type RequestAnalysis struct {
+ // API is the classification of the endpoint.
+ API MaskableAPI
+ // Index is the target index name extracted from the URL (empty if not applicable).
+ Index string
+ // BlockedFeatures lists features found in the request body that must be blocked.
+ BlockedFeatures []BlockedFeature
+ // SourceFields is the list of fields requested via _source filtering (nil = all fields).
+ // An empty slice means _source is disabled.
+ SourceFields []string
+ // SourceDisabled is true when _source: false is set.
+ SourceDisabled bool
+ // RequestedFields is the list of fields requested via the "fields" parameter.
+ RequestedFields []string
+ // HighlightFields is the list of fields requested via "highlight.fields".
+ HighlightFields []string
+ // SortFields is the list of field paths used in "sort".
+ SortFields []string
+ // HasInnerHits is true if the query contains inner_hits.
+ HasInnerHits bool
+ // PredicateFields is the list of field paths used in query predicates.
+ PredicateFields []string
+}
+
+// blockedEndpoints lists URL patterns that must be rejected when masking is active.
+// These overlap with read patterns but cannot be safely masked.
+// Includes both Elasticsearch and OpenSearch equivalents.
+var blockedEndpoints = []string{
+ // Elasticsearch endpoints.
+ "_async_search",
+ "_search/scroll",
+ "_search_template",
+ "_msearch/template",
+ "_sql",
+ "_eql/search",
+ "_esql/query",
+ "_terms_enum",
+ "_termvectors",
+ "_mtermvectors",
+ "_knn_search",
+ // OpenSearch equivalents (plugin-based paths).
+ "_plugins/_asynchronous_search",
+ "_plugins/_sql",
+ "_plugins/_ppl",
+}
+
+// classifyMaskableAPI determines the MaskableAPI type and target index for an ES request.
+func classifyMaskableAPI(method, url string) (MaskableAPI, string) {
+ method = strings.ToUpper(method)
+
+ // Strip query parameters.
+ path := url
+ if i := strings.IndexByte(path, '?'); i >= 0 {
+ path = path[:i]
+ }
+
+ // Strip leading slash.
+ path = strings.TrimPrefix(path, "/")
+
+ // Only GET and POST can return user data.
+ switch method {
+ case "GET":
+ if blocked, index := isBlockedEndpoint(path); blocked {
+ return APIBlocked, index
+ }
+ return classifyGetForMasking(path)
+ case "POST":
+ if blocked, index := isBlockedEndpoint(path); blocked {
+ return APIBlocked, index
+ }
+ return classifyPostForMasking(path)
+ default:
+ // HEAD, PUT, DELETE, PATCH do not return user data.
+ return APIUnsupported, ""
+ }
+}
+
+// isBlockedEndpoint checks whether the path matches a blocked endpoint pattern.
+// Returns true and the extracted index if blocked.
+func isBlockedEndpoint(path string) (bool, string) {
+ for _, pattern := range blockedEndpoints {
+ if strings.Contains(path, pattern) {
+ return true, extractIndex(path)
+ }
+ }
+ return false, ""
+}
+
+// classifyGetForMasking classifies a GET request path for masking.
+func classifyGetForMasking(path string) (MaskableAPI, string) {
+ for part := range strings.SplitSeq(path, "/") {
+ switch part {
+ case "_search", "_msearch":
+ return APIMaskSearch, extractIndex(path)
+ case "_doc":
+ return APIMaskGetDoc, extractIndex(path)
+ case "_source":
+ return APIMaskGetSource, extractIndex(path)
+ case "_mget":
+ return APIMaskMGet, extractIndex(path)
+ }
+ }
+
+ return APIUnsupported, ""
+}
+
+// classifyPostForMasking classifies a POST request path for masking.
+func classifyPostForMasking(path string) (MaskableAPI, string) {
+ for part := range strings.SplitSeq(path, "/") {
+ switch part {
+ case "_search", "_msearch":
+ return APIMaskSearch, extractIndex(path)
+ case "_mget":
+ return APIMaskMGet, extractIndex(path)
+ case "_explain":
+ return APIMaskExplain, extractIndex(path)
+ }
+ }
+
+ return APIUnsupported, ""
+}
+
+// extractIndex returns the first path segment if it does not start with "_".
+// This extracts the index name from ES URL paths like "<index>/_search".
+func extractIndex(path string) string {
+ if path == "" {
+ return ""
+ }
+ parts := strings.SplitN(path, "/", 2)
+ if len(parts) == 0 {
+ return ""
+ }
+ first := parts[0]
+ if strings.HasPrefix(first, "_") {
+ return ""
+ }
+ return first
+}
+
+// AnalyzeRequest combines URL classification with body analysis for an ES request.
+// Body analysis is only performed for search and explain APIs.
+func AnalyzeRequest(method, url, body string) *RequestAnalysis {
+ api, index := classifyMaskableAPI(method, url)
+ result := &RequestAnalysis{
+ API: api,
+ Index: index,
+ }
+
+ // Only analyze the body for APIs that accept a query body.
+ switch api {
+ case APIMaskSearch, APIMaskExplain:
+ bodyResult := analyzeRequestBody(body)
+ result.BlockedFeatures = bodyResult.BlockedFeatures
+ result.SourceFields = bodyResult.SourceFields
+ result.SourceDisabled = bodyResult.SourceDisabled
+ result.RequestedFields = bodyResult.RequestedFields
+ result.HighlightFields = bodyResult.HighlightFields
+ result.SortFields = bodyResult.SortFields
+ result.HasInnerHits = bodyResult.HasInnerHits
+ result.PredicateFields = bodyResult.PredicateFields
+ default:
+ // No body analysis for other API types.
+ }
+
+ return result
+}
+
+// analyzeRequestBody parses the JSON body and extracts fields and blocked features.
+func analyzeRequestBody(body string) *RequestAnalysis {
+ result := &RequestAnalysis{}
+
+ parsed := make(map[string]any)
+ if json.Unmarshal([]byte(body), &parsed) != nil {
+ return result
+ }
+
+ result.BlockedFeatures = detectBlockedFeatures(parsed)
+ extractSource(parsed, result)
+ result.RequestedFields = extractStringArray(parsed, "fields")
+ result.HighlightFields = extractHighlightFields(parsed)
+ result.SortFields = extractSortFields(parsed)
+ result.HasInnerHits = containsKey(parsed, "inner_hits")
+ result.PredicateFields = extractPredicateFields(parsed)
+
+ return result
+}
+
+// detectBlockedFeatures checks for top-level keys that indicate blocked features.
+func detectBlockedFeatures(parsed map[string]any) []BlockedFeature {
+ var blocked []BlockedFeature
+
+ if _, ok := parsed["aggs"]; ok {
+ blocked = append(blocked, BlockedFeatureAggs)
+ } else if _, ok := parsed["aggregations"]; ok {
+ blocked = append(blocked, BlockedFeatureAggs)
+ }
+ if _, ok := parsed["suggest"]; ok {
+ blocked = append(blocked, BlockedFeatureSuggest)
+ }
+ if _, ok := parsed["script_fields"]; ok {
+ blocked = append(blocked, BlockedFeatureScriptFields)
+ }
+ if _, ok := parsed["runtime_mappings"]; ok {
+ blocked = append(blocked, BlockedFeatureRuntimeMappings)
+ }
+ if _, ok := parsed["stored_fields"]; ok {
+ blocked = append(blocked, BlockedFeatureStoredFields)
+ }
+ if _, ok := parsed["docvalue_fields"]; ok {
+ blocked = append(blocked, BlockedFeatureDocvalueFields)
+ }
+
+ return blocked
+}
+
+// extractSource handles the three forms of _source in the request body.
+func extractSource(parsed map[string]any, result *RequestAnalysis) {
+ src, ok := parsed["_source"]
+ if !ok {
+ return
+ }
+
+ switch v := src.(type) {
+ case bool:
+ if !v {
+ result.SourceDisabled = true
+ }
+ case []any:
+ result.SourceFields = toStringSlice(v)
+ case map[string]any:
+ if includes, ok := v["includes"]; ok {
+ if arr, ok := includes.([]any); ok {
+ result.SourceFields = toStringSlice(arr)
+ }
+ }
+ }
+}
+
+// extractStringArray extracts a string array value from a top-level key.
+func extractStringArray(parsed map[string]any, key string) []string {
+ val, ok := parsed[key]
+ if !ok {
+ return nil
+ }
+ arr, ok := val.([]any)
+ if !ok {
+ return nil
+ }
+ return toStringSlice(arr)
+}
+
+// extractHighlightFields extracts field names from highlight.fields.
+func extractHighlightFields(parsed map[string]any) []string {
+ highlight, ok := parsed["highlight"]
+ if !ok {
+ return nil
+ }
+ hlMap, ok := highlight.(map[string]any)
+ if !ok {
+ return nil
+ }
+ fields, ok := hlMap["fields"]
+ if !ok {
+ return nil
+ }
+ fieldsMap, ok := fields.(map[string]any)
+ if !ok {
+ return nil
+ }
+ result := make([]string, 0, len(fieldsMap))
+ for k := range fieldsMap {
+ result = append(result, k)
+ }
+ slices.Sort(result)
+ return result
+}
+
+// extractSortFields extracts field names from the sort array.
+func extractSortFields(parsed map[string]any) []string {
+ sortVal, ok := parsed["sort"]
+ if !ok {
+ return nil
+ }
+ arr, ok := sortVal.([]any)
+ if !ok {
+ return nil
+ }
+ var fields []string
+ for _, item := range arr {
+ switch v := item.(type) {
+ case string:
+ if strings.HasPrefix(v, "_") {
+ continue
+ }
+ fields = append(fields, v)
+ case map[string]any:
+ for k := range v {
+ if !strings.HasPrefix(k, "_") {
+ fields = append(fields, k)
+ }
+ }
+ }
+ }
+ return fields
+}
+
+// containsKey recursively searches for a key in a nested map structure.
+func containsKey(data map[string]any, key string) bool {
+ for k, v := range data {
+ if k == key {
+ return true
+ }
+ if containsKeyInValue(v, key) {
+ return true
+ }
+ }
+ return false
+}
+
+// containsKeyInValue checks if any nested map within v contains the given key.
+func containsKeyInValue(v any, key string) bool {
+ switch child := v.(type) {
+ case map[string]any:
+ return containsKey(child, key)
+ case []any:
+ for _, item := range child {
+ if m, ok := item.(map[string]any); ok && containsKey(m, key) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// toStringSlice converts a []any to a []string, skipping non-string elements.
+func toStringSlice(arr []any) []string {
+ result := make([]string, 0, len(arr))
+ for _, item := range arr {
+ if s, ok := item.(string); ok {
+ result = append(result, s)
+ }
+ }
+ return result
+}
+
+// directFieldClauses are clause types where the keys inside the clause object are field names.
+var directFieldClauses = map[string]bool{
+ "match": true, "match_phrase": true, "match_phrase_prefix": true, "match_bool_prefix": true,
+ "term": true, "terms": true, "terms_set": true,
+ "range": true,
+ "wildcard": true, "prefix": true, "fuzzy": true, "regexp": true,
+ "geo_distance": true, "geo_bounding_box": true, "geo_shape": true, "geo_polygon": true,
+ "span_term": true,
+}
+
+// compoundClauses maps clause types to the keys that contain sub-queries.
+var compoundClauses = map[string][]string{
+ "bool": {"must", "must_not", "should", "filter"},
+ "nested": {"query"},
+ "has_child": {"query"},
+ "has_parent": {"query"},
+ "dis_max": {"queries"},
+ "constant_score": {"filter"},
+ "boosting": {"positive", "negative"},
+ "function_score": {"query"},
+}
+
+// extractPredicateFields extracts field names from ES query clauses.
+func extractPredicateFields(parsed map[string]any) []string {
+ queryVal, ok := parsed["query"]
+ if !ok {
+ return nil
+ }
+ queryMap, ok := queryVal.(map[string]any)
+ if !ok {
+ return nil
+ }
+ var fields []string
+ extractFieldsFromQuery(queryMap, &fields)
+ return fields
+}
+
+// extractFieldsFromQuery recursively walks ES query clauses to collect field names.
+func extractFieldsFromQuery(queryMap map[string]any, fields *[]string) {
+ for clauseType, clauseVal := range queryMap {
+ clauseObj, ok := clauseVal.(map[string]any)
+ if !ok {
+ extractFieldsFromQueryArray(clauseVal, fields)
+ continue
+ }
+ switch {
+ case directFieldClauses[clauseType]:
+ extractDirectFieldNames(clauseObj, fields)
+ case clauseType == "exists":
+ if fieldVal, ok := clauseObj["field"].(string); ok {
+ *fields = append(*fields, fieldVal)
+ }
+ case compoundClauses[clauseType] != nil:
+ extractCompoundClauseFields(clauseObj, compoundClauses[clauseType], fields)
+ default:
+ extractFieldsFromQuery(clauseObj, fields)
+ }
+ }
+}
+
+// extractFieldsFromQueryArray handles the case where a clause value is an array of query maps.
+func extractFieldsFromQueryArray(val any, fields *[]string) {
+ arr, ok := val.([]any)
+ if !ok {
+ return
+ }
+ for _, item := range arr {
+ if m, ok := item.(map[string]any); ok {
+ extractFieldsFromQuery(m, fields)
+ }
+ }
+}
+
+// extractDirectFieldNames extracts field names from a direct field clause object.
+func extractDirectFieldNames(clauseObj map[string]any, fields *[]string) {
+ for fieldName := range clauseObj {
+ if !strings.HasPrefix(fieldName, "_") && fieldName != "boost" {
+ *fields = append(*fields, fieldName)
+ }
+ }
+}
+
+// extractCompoundClauseFields recurses into sub-query keys of a compound clause.
+func extractCompoundClauseFields(clauseObj map[string]any, subQueryKeys []string, fields *[]string) {
+ for _, sqKey := range subQueryKeys {
+ sqVal, ok := clauseObj[sqKey]
+ if !ok {
+ continue
+ }
+ switch sq := sqVal.(type) {
+ case map[string]any:
+ extractFieldsFromQuery(sq, fields)
+ case []any:
+ extractFieldsFromQueryArray(sq, fields)
+ }
+ }
+}
diff --git a/backend/plugin/parser/elasticsearch/masking_test.go b/backend/plugin/parser/elasticsearch/masking_test.go
new file mode 100644
index 0000000000..3a0ee219d3
--- /dev/null
+++ b/backend/plugin/parser/elasticsearch/masking_test.go
@@ -0,0 +1,157 @@
+package elasticsearch
+
+import (
+ "io"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "gopkg.in/yaml.v3"
+)
+
+func TestClassifyMaskableAPI(t *testing.T) {
+ type testCase struct {
+ Description string `yaml:"description"`
+ Method string `yaml:"method"`
+ URL string `yaml:"url"`
+ WantAPI MaskableAPI `yaml:"wantAPI"`
+ WantIndex string `yaml:"wantIndex"`
+ }
+
+ a := require.New(t)
+ data, err := os.Open("test-data/masking_classify_api.yaml")
+ a.NoError(err)
+ raw, err := io.ReadAll(data)
+ a.NoError(err)
+ a.NoError(data.Close())
+
+ var cases []testCase
+ a.NoError(yaml.Unmarshal(raw, &cases))
+
+ for _, tc := range cases {
+ t.Run(tc.Description, func(t *testing.T) {
+ api, index := classifyMaskableAPI(tc.Method, tc.URL)
+ require.Equal(t, tc.WantAPI, api, "API classification")
+ require.Equal(t, tc.WantIndex, index, "index extraction")
+ })
+ }
+}
+
+func TestAnalyzeRequestBody(t *testing.T) {
+ type testCase struct {
+ Description string `yaml:"description"`
+ Body string `yaml:"body"`
+ WantBlocked []BlockedFeature `yaml:"wantBlocked"`
+ WantSourceDisabled bool `yaml:"wantSourceDisabled"`
+ WantSourceFields []string `yaml:"wantSourceFields"`
+ WantFields []string `yaml:"wantFields"`
+ WantHighlight []string `yaml:"wantHighlight"`
+ WantSort []string `yaml:"wantSort"`
+ WantInnerHits bool `yaml:"wantInnerHits"`
+ }
+
+ a := require.New(t)
+ data, err := os.Open("test-data/masking_analyze_body.yaml")
+ a.NoError(err)
+ raw, err := io.ReadAll(data)
+ a.NoError(err)
+ a.NoError(data.Close())
+
+ var cases []testCase
+ a.NoError(yaml.Unmarshal(raw, &cases))
+
+ for _, tc := range cases {
+ t.Run(tc.Description, func(t *testing.T) {
+ result := analyzeRequestBody(tc.Body)
+ if tc.WantBlocked != nil {
+ require.Equal(t, tc.WantBlocked, result.BlockedFeatures)
+ } else {
+ require.Empty(t, result.BlockedFeatures)
+ }
+ require.Equal(t, tc.WantSourceDisabled, result.SourceDisabled)
+ if tc.WantSourceFields != nil {
+ require.Equal(t, tc.WantSourceFields, result.SourceFields)
+ }
+ if tc.WantFields != nil {
+ require.Equal(t, tc.WantFields, result.RequestedFields)
+ }
+ if tc.WantHighlight != nil {
+ require.ElementsMatch(t, tc.WantHighlight, result.HighlightFields)
+ }
+ if tc.WantSort != nil {
+ require.Equal(t, tc.WantSort, result.SortFields)
+ }
+ require.Equal(t, tc.WantInnerHits, result.HasInnerHits)
+ })
+ }
+}
+
+func TestAnalyzeRequest(t *testing.T) {
+ type testCase struct {
+ Description string `yaml:"description"`
+ Method string `yaml:"method"`
+ URL string `yaml:"url"`
+ Body string `yaml:"body"`
+ WantAPI MaskableAPI `yaml:"wantAPI"`
+ WantIndex string `yaml:"wantIndex"`
+ WantBlocked []BlockedFeature `yaml:"wantBlocked"`
+ WantPredicateFields []string `yaml:"wantPredicateFields"`
+ }
+
+ a := require.New(t)
+ data, err := os.Open("test-data/masking_analyze_request.yaml")
+ a.NoError(err)
+ raw, err := io.ReadAll(data)
+ a.NoError(err)
+ a.NoError(data.Close())
+
+ var cases []testCase
+ a.NoError(yaml.Unmarshal(raw, &cases))
+
+ for _, tc := range cases {
+ t.Run(tc.Description, func(t *testing.T) {
+ result := AnalyzeRequest(tc.Method, tc.URL, tc.Body)
+ require.Equal(t, tc.WantAPI, result.API)
+ if tc.WantIndex != "" {
+ require.Equal(t, tc.WantIndex, result.Index)
+ }
+ if tc.WantBlocked != nil {
+ require.Equal(t, tc.WantBlocked, result.BlockedFeatures)
+ } else {
+ require.Empty(t, result.BlockedFeatures)
+ }
+ if tc.WantPredicateFields != nil {
+ require.ElementsMatch(t, tc.WantPredicateFields, result.PredicateFields)
+ }
+ })
+ }
+}
+
+func TestExtractPredicateFields(t *testing.T) {
+ type testCase struct {
+ Description string `yaml:"description"`
+ Body string `yaml:"body"`
+ WantFields []string `yaml:"wantFields"`
+ }
+
+ a := require.New(t)
+ data, err := os.Open("test-data/masking_predicate_fields.yaml")
+ a.NoError(err)
+ raw, err := io.ReadAll(data)
+ a.NoError(err)
+ a.NoError(data.Close())
+
+ var cases []testCase
+ a.NoError(yaml.Unmarshal(raw, &cases))
+
+ for _, tc := range cases {
+ t.Run(tc.Description, func(t *testing.T) {
+ result := analyzeRequestBody(tc.Body)
+ if len(tc.WantFields) > 0 {
+ require.ElementsMatch(t, tc.WantFields, result.PredicateFields)
+ } else {
+ require.Empty(t, result.PredicateFields)
+ }
+ })
+ }
+}
diff --git a/backend/plugin/parser/elasticsearch/test-data/masking_analyze_body.yaml b/backend/plugin/parser/elasticsearch/test-data/masking_analyze_body.yaml
new file mode 100644
index 0000000000..89a58d1e78
--- /dev/null
+++ b/backend/plugin/parser/elasticsearch/test-data/masking_analyze_body.yaml
@@ -0,0 +1,100 @@
+# Test cases for analyzeRequestBody.
+# BlockedFeature enum: BlockedFeatureAggs=0, BlockedFeatureSuggest=1, BlockedFeatureScriptFields=2,
+# BlockedFeatureRuntimeMappings=3, BlockedFeatureStoredFields=4, BlockedFeatureDocvalueFields=5
+- description: "empty body"
+ body: "{}"
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "simple query only"
+ body: '{"query": {"match_all": {}}}'
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "blocked: aggregations with aggs key"
+ body: '{"query": {"match_all": {}}, "aggs": {"status_count": {"terms": {"field": "status"}}}}'
+ wantBlocked:
+ - 0
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "blocked: aggregations with aggregations key"
+ body: '{"aggregations": {"status_count": {"terms": {"field": "status"}}}}'
+ wantBlocked:
+ - 0
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "blocked: suggest"
+ body: '{"suggest": {"my-suggest": {"text": "test", "term": {"field": "title"}}}}'
+ wantBlocked:
+ - 1
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "blocked: script_fields"
+ body: '{"script_fields": {"my_field": {"script": "doc[''price''].value * 2"}}}'
+ wantBlocked:
+ - 2
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "blocked: runtime_mappings"
+ body: '{"runtime_mappings": {"day_of_week": {"type": "keyword"}}}'
+ wantBlocked:
+ - 3
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "blocked: stored_fields"
+ body: '{"stored_fields": ["title"]}'
+ wantBlocked:
+ - 4
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "blocked: docvalue_fields"
+ body: '{"docvalue_fields": ["date"]}'
+ wantBlocked:
+ - 5
+ wantSourceDisabled: false
+ wantInnerHits: false
+- description: "source disabled"
+ body: '{"_source": false}'
+ wantSourceDisabled: true
+ wantInnerHits: false
+- description: "source field list"
+ body: '{"_source": ["name", "email"]}'
+ wantSourceDisabled: false
+ wantSourceFields:
+ - "name"
+ - "email"
+ wantInnerHits: false
+- description: "source includes/excludes"
+ body: '{"_source": {"includes": ["name", "contact.*"]}}'
+ wantSourceDisabled: false
+ wantSourceFields:
+ - "name"
+ - "contact.*"
+ wantInnerHits: false
+- description: "fields parameter"
+ body: '{"fields": ["created_at", "status"]}'
+ wantSourceDisabled: false
+ wantFields:
+ - "created_at"
+ - "status"
+ wantInnerHits: false
+- description: "highlight fields"
+ body: '{"highlight": {"fields": {"message": {}, "title": {}}}}'
+ wantSourceDisabled: false
+ wantHighlight:
+ - "message"
+ - "title"
+ wantInnerHits: false
+- description: "sort fields"
+ body: '{"sort": [{"date": "desc"}, "_score", {"price": {"order": "asc"}}]}'
+ wantSourceDisabled: false
+ wantSort:
+ - "date"
+ - "price"
+ wantInnerHits: false
+- description: "inner_hits present in nested query"
+ body: '{"query": {"nested": {"path": "comments", "inner_hits": {}}}}'
+ wantSourceDisabled: false
+ wantInnerHits: true
+- description: "inner_hits absent"
+ body: '{"query": {"match": {"title": "test"}}}'
+ wantSourceDisabled: false
+ wantInnerHits: false
diff --git a/backend/plugin/parser/elasticsearch/test-data/masking_analyze_request.yaml b/backend/plugin/parser/elasticsearch/test-data/masking_analyze_request.yaml
new file mode 100644
index 0000000000..0ea4d275df
--- /dev/null
+++ b/backend/plugin/parser/elasticsearch/test-data/masking_analyze_request.yaml
@@ -0,0 +1,27 @@
+# Test cases for AnalyzeRequest.
+# MaskableAPI enum: APIUnsupported=0, APIMaskSearch=1, APIMaskGetDoc=2, APIMaskGetSource=3, APIMaskMGet=4, APIMaskExplain=5, APIBlocked=6
+- description: "search with blocked feature"
+ method: "POST"
+ url: "/my-index/_search"
+ body: '{"aggs": {"x": {"terms": {"field": "status"}}}}'
+ wantAPI: 1
+ wantIndex: "my-index"
+ wantBlocked:
+ - 0
+- description: "get doc skips body analysis"
+ method: "GET"
+ url: "/my-index/_doc/1"
+ body: '{"aggs": {"x": {}}}'
+ wantAPI: 2
+- description: "blocked endpoint"
+ method: "POST"
+ url: "/_sql"
+ body: '{"query": "SELECT * FROM test"}'
+ wantAPI: 6
+- description: "search copies predicate fields"
+ method: "POST"
+ url: "/my-index/_search"
+ body: '{"query": {"match": {"email": "[email protected]"}}}'
+ wantAPI: 1
+ wantPredicateFields:
+ - "email"
diff --git a/backend/plugin/parser/elasticsearch/test-data/masking_classify_api.yaml b/backend/plugin/parser/elasticsearch/test-data/masking_classify_api.yaml
new file mode 100644
index 0000000000..8bc8b4b089
--- /dev/null
+++ b/backend/plugin/parser/elasticsearch/test-data/masking_classify_api.yaml
@@ -0,0 +1,180 @@
+# Test cases for classifyMaskableAPI.
+# MaskableAPI enum: APIUnsupported=0, APIMaskSearch=1, APIMaskGetDoc=2, APIMaskGetSource=3, APIMaskMGet=4, APIMaskExplain=5, APIBlocked=6
+
+# Supported: _search
+- description: "POST _search"
+ method: "POST"
+ url: "/my-index/_search"
+ wantAPI: 1
+ wantIndex: "my-index"
+- description: "GET _search"
+ method: "GET"
+ url: "/my-index/_search"
+ wantAPI: 1
+ wantIndex: "my-index"
+- description: "POST _search with query params"
+ method: "POST"
+ url: "/my-index/_search?size=10"
+ wantAPI: 1
+ wantIndex: "my-index"
+
+# Supported: _msearch
+- description: "POST _msearch without index"
+ method: "POST"
+ url: "/_msearch"
+ wantAPI: 1
+ wantIndex: ""
+- description: "POST _msearch with index"
+ method: "POST"
+ url: "/my-index/_msearch"
+ wantAPI: 1
+ wantIndex: "my-index"
+
+# Supported: GET _doc
+- description: "GET _doc"
+ method: "GET"
+ url: "/my-index/_doc/1"
+ wantAPI: 2
+ wantIndex: "my-index"
+
+# Supported: GET _source
+- description: "GET _source"
+ method: "GET"
+ url: "/my-index/_source/1"
+ wantAPI: 3
+ wantIndex: "my-index"
+
+# Supported: _mget
+- description: "POST _mget without index"
+ method: "POST"
+ url: "/_mget"
+ wantAPI: 4
+ wantIndex: ""
+- description: "POST _mget with index"
+ method: "POST"
+ url: "/my-index/_mget"
+ wantAPI: 4
+ wantIndex: "my-index"
+- description: "GET _mget without index"
+ method: "GET"
+ url: "/_mget"
+ wantAPI: 4
+ wantIndex: ""
+
+# Supported: _explain
+- description: "POST _explain"
+ method: "POST"
+ url: "/my-index/_explain/1"
+ wantAPI: 5
+ wantIndex: "my-index"
+
+# Blocked endpoints
+- description: "POST _async_search blocked"
+ method: "POST"
+ url: "/my-index/_async_search"
+ wantAPI: 6
+ wantIndex: "my-index"
+- description: "GET _async_search status blocked"
+ method: "GET"
+ url: "/_async_search/abc123"
+ wantAPI: 6
+ wantIndex: ""
+- description: "POST _search/scroll blocked"
+ method: "POST"
+ url: "/_search/scroll"
+ wantAPI: 6
+ wantIndex: ""
+- description: "POST _search_template blocked"
+ method: "POST"
+ url: "/my-index/_search_template"
+ wantAPI: 6
+ wantIndex: "my-index"
+- description: "POST _msearch/template blocked"
+ method: "POST"
+ url: "/_msearch/template"
+ wantAPI: 6
+ wantIndex: ""
+- description: "POST _sql blocked"
+ method: "POST"
+ url: "/_sql"
+ wantAPI: 6
+ wantIndex: ""
+- description: "POST _eql/search blocked"
+ method: "POST"
+ url: "/my-index/_eql/search"
+ wantAPI: 6
+ wantIndex: "my-index"
+- description: "POST _esql/query blocked"
+ method: "POST"
+ url: "/_esql/query"
+ wantAPI: 6
+ wantIndex: ""
+- description: "POST _terms_enum blocked"
+ method: "POST"
+ url: "/my-index/_terms_enum"
+ wantAPI: 6
+ wantIndex: "my-index"
+- description: "GET _termvectors blocked"
+ method: "GET"
+ url: "/my-index/_termvectors/1"
+ wantAPI: 6
+ wantIndex: "my-index"
+
+# OpenSearch blocked endpoints
+- description: "POST _plugins/_asynchronous_search blocked (OpenSearch)"
+ method: "POST"
+ url: "/_plugins/_asynchronous_search"
+ wantAPI: 6
+ wantIndex: ""
+- description: "POST _plugins/_sql blocked (OpenSearch)"
+ method: "POST"
+ url: "/_plugins/_sql"
+ wantAPI: 6
+ wantIndex: ""
+- description: "POST _plugins/_ppl blocked (OpenSearch)"
+ method: "POST"
+ url: "/_plugins/_ppl"
+ wantAPI: 6
+ wantIndex: ""
+
+# Unsupported (no user data)
+- description: "HEAD _doc unsupported"
+ method: "HEAD"
+ url: "/my-index/_doc/1"
+ wantAPI: 0
+ wantIndex: ""
+- description: "PUT _doc unsupported"
+ method: "PUT"
+ url: "/my-index/_doc/1"
+ wantAPI: 0
+ wantIndex: ""
+- description: "DELETE _doc unsupported"
+ method: "DELETE"
+ url: "/my-index/_doc/1"
+ wantAPI: 0
+ wantIndex: ""
+- description: "GET _cat/indices unsupported"
+ method: "GET"
+ url: "/_cat/indices"
+ wantAPI: 0
+ wantIndex: ""
+- description: "GET _cluster/health unsupported"
+ method: "GET"
+ url: "/_cluster/health"
+ wantAPI: 0
+ wantIndex: ""
+- description: "POST _count unsupported"
+ method: "POST"
+ url: "/my-index/_count"
+ wantAPI: 0
+ wantIndex: ""
+- description: "POST _validate/query unsupported"
+ method: "POST"
+ url: "/my-index/_validate/query"
+ wantAPI: 0
+ wantIndex: ""
+- description: "PUT index unsupported"
+ method: "PUT"
+ url: "/my-index"
+ wantAPI: 0
+ wantIndex: ""
diff --git a/backend/plugin/parser/elasticsearch/test-data/masking_predicate_fields.yaml b/backend/plugin/parser/elasticsearch/test-data/masking_predicate_fields.yaml
new file mode 100644
index 0000000000..ec728976e8
--- /dev/null
+++ b/backend/plugin/parser/elasticsearch/test-data/masking_predicate_fields.yaml
@@ -0,0 +1,36 @@
+# Test cases for extractPredicateFields (via analyzeRequestBody).
+- description: "match query"
+ body: '{"query": {"match": {"email": "[email protected]"}}}'
+ wantFields:
+ - "email"
+- description: "term query"
+ body: '{"query": {"term": {"status": "active"}}}'
+ wantFields:
+ - "status"
+- description: "range query"
+ body: '{"query": {"range": {"age": {"gte": 18}}}}'
+ wantFields:
+ - "age"
+- description: "bool with must and filter"
+ body: '{"query": {"bool": {"must": [{"match": {"name": "Alice"}}], "filter": [{"term": {"status": "active"}}]}}}'
+ wantFields:
+ - "name"
+ - "status"
+- description: "exists query"
+ body: '{"query": {"exists": {"field": "email"}}}'
+ wantFields:
+ - "email"
+- description: "nested query"
+ body: '{"query": {"nested": {"path": "comments", "query": {"match": {"comments.text": "hello"}}}}}'
+ wantFields:
+ - "comments.text"
+- description: "no query clause"
+ body: '{"size": 10}'
+ wantFields: []
+- description: "match_all"
+ body: '{"query": {"match_all": {}}}'
+ wantFields: []
+- description: "wildcard query"
+ body: '{"query": {"wildcard": {"name": {"value": "ali*"}}}}'
+ wantFields:
+ - "name"
diff --git a/frontend/src/utils/v1/instance.ts b/frontend/src/utils/v1/instance.ts
index 1f20a0f663..b163d0964b 100644
--- a/frontend/src/utils/v1/instance.ts
+++ b/frontend/src/utils/v1/instance.ts
@@ -299,7 +299,9 @@ export const instanceV1MaskingForNoSQL = (
instanceOrEngine: Instance | InstanceResource | Engine
) => {
const engine = engineOfInstanceV1(instanceOrEngine);
- return [Engine.MONGODB, Engine.COSMOSDB].includes(engine);
+ return [Engine.MONGODB, Engine.COSMOSDB, Engine.ELASTICSEARCH].includes(
+ engine
+ );
};
export const engineOfInstanceV1 = (