Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: implement data masking for Elasticsearch/OpenSearch queries (#19394)
h3n4l41921b809557
diff --git a/backend/api/v1/catalog_masking.go b/backend/api/v1/catalog_masking.goindex c75ab7e79b..e92eb0d559 100644--- a/backend/api/v1/catalog_masking.go+++ b/backend/api/v1/catalog_masking.go@@ -110,11 +110,7 @@ func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*base.PathAST, oo, parentSemanticType = getObjectSchemaByPath(o, ast)if parentSemanticType != "" {if m, ok := semanticTypeToMasker[parentSemanticType]; ok {- maskedData, err := applyMaskerToData(value, m)- if err != nil {- return nil, err- }- result[key] = maskedData+ result[key] = getJSONMemberFromRowValue(m.Mask(nil))continue}}@@ -182,13 +178,9 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, semaswitch data := data.(type) {case map[string]any:if objectSchema.SemanticType != "" {- // If the outer semantic type is found, apply the masker recursively to the object.+ // If the semantic type is found, replace the entire value directly.if m, ok := semanticTypeToMasker[objectSchema.SemanticType]; ok {- maskedData, err := applyMaskerToData(data, m)- if err != nil {- return nil, err- }- return maskedData, nil+ return getJSONMemberFromRowValue(m.Mask(nil)), nil}} else {// Otherwise, recursively walk the object.@@ -211,13 +203,9 @@ func walkAndMaskJSONRecursive(data any, objectSchema *storepb.ObjectSchema, semareturn data, nilcase []any:if objectSchema.SemanticType != "" {- // If the outer semantic type is found, apply the masker recursively to the array.+ // If the semantic type is found, replace the entire value directly.if m, ok := semanticTypeToMasker[objectSchema.SemanticType]; ok {- maskedData, err := applyMaskerToData(data, m)- if err != nil {- return nil, err- }- return maskedData, nil+ return getJSONMemberFromRowValue(m.Mask(nil)), nil}} else {arrayKind := objectSchema.GetArrayKind()diff --git a/backend/api/v1/catalog_masking_elasticsearch.go b/backend/api/v1/catalog_masking_elasticsearch.gonew file mode 100644index 0000000000..bbc49f56d4--- /dev/null+++ b/backend/api/v1/catalog_masking_elasticsearch.go@@ -0,0 +1,484 @@+package v1++import (+ "context"+ "encoding/json"+ "strings"++ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/component/masker"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"+ "github.com/bytebase/bytebase/backend/store"+)++// lookupSemanticTypeByDotPath splits a dot-delimited field path and walks the ObjectSchema+// to find the semantic type at the leaf.+func lookupSemanticTypeByDotPath(dotPath string, objectSchema *storepb.ObjectSchema) string {+ parts := strings.Split(dotPath, ".")+ current := objectSchema+ for _, part := range parts {+ if current == nil {+ return ""+ }+ if current.SemanticType != "" {+ return current.SemanticType+ }+ structKind := current.GetStructKind()+ if structKind == nil {+ return ""+ }+ props := structKind.GetProperties()+ if props == nil {+ return ""+ }+ child, ok := props[part]+ if !ok {+ return ""+ }+ current = child+ }+ if current != nil {+ return current.SemanticType+ }+ return ""+}++// maskElasticsearchSourceObject masks a single _source JSON object using+// the ObjectSchema and semantic type maskers. It delegates to walkAndMaskJSONRecursive+// from catalog_masking.go: when a field has a semantic type, the entire value is+// replaced directly regardless of whether it is a primitive, object, or array.+func maskElasticsearchSourceObject(source map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {+ if objectSchema == nil {+ return source, nil+ }+ masked, err := walkAndMaskJSONRecursive(source, objectSchema, semanticTypeToMasker)+ if err != nil {+ return nil, err+ }+ result, ok := masked.(map[string]any)+ if !ok {+ return source, nil+ }+ return result, nil+}++// maskElasticsearchHitFields masks the "fields" section of a hit.+// Fields use dot-path keys with array values.+func maskElasticsearchHitFields(hitMap map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {+ fieldsVal, ok := hitMap["fields"]+ if !ok {+ return nil+ }+ fieldsMap, ok := fieldsVal.(map[string]any)+ if !ok {+ return nil+ }+ for dotPath, values := range fieldsMap {+ semanticType := lookupSemanticTypeByDotPath(dotPath, objectSchema)+ if semanticType == "" {+ continue+ }+ m, ok := semanticTypeToMasker[semanticType]+ if !ok {+ continue+ }+ arr, ok := values.([]any)+ if !ok {+ continue+ }+ for i, val := range arr {+ masked, err := applyMaskerToData(val, m)+ if err != nil {+ return err+ }+ arr[i] = masked+ }+ fieldsMap[dotPath] = arr+ }+ hitMap["fields"] = fieldsMap+ return nil+}++// maskElasticsearchHitHighlight masks the "highlight" section of a hit.+// Highlight uses dot-path keys with array values (HTML fragment strings).+func maskElasticsearchHitHighlight(hitMap map[string]any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {+ highlightVal, ok := hitMap["highlight"]+ if !ok {+ return nil+ }+ highlightMap, ok := highlightVal.(map[string]any)+ if !ok {+ return nil+ }+ for dotPath, values := range highlightMap {+ semanticType := lookupSemanticTypeByDotPath(dotPath, objectSchema)+ if semanticType == "" {+ continue+ }+ m, ok := semanticTypeToMasker[semanticType]+ if !ok {+ continue+ }+ arr, ok := values.([]any)+ if !ok {+ continue+ }+ for i, val := range arr {+ masked, err := applyMaskerToData(val, m)+ if err != nil {+ return err+ }+ arr[i] = masked+ }+ highlightMap[dotPath] = arr+ }+ hitMap["highlight"] = highlightMap+ return nil+}++// maskElasticsearchHitSort masks the "sort" array of a hit based on the sort fields from the request.+// sortFields is the ordered list of field names used in the sort clause (from request analysis).+func maskElasticsearchHitSort(hitMap map[string]any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {+ sortVal, ok := hitMap["sort"]+ if !ok {+ return nil+ }+ sortArr, ok := sortVal.([]any)+ if !ok {+ return nil+ }+ for i, val := range sortArr {+ if i >= len(sortFields) {+ break+ }+ fieldName := sortFields[i]+ if fieldName == "" {+ continue+ }+ semanticType := lookupSemanticTypeByDotPath(fieldName, objectSchema)+ if semanticType == "" {+ continue+ }+ m, ok := semanticTypeToMasker[semanticType]+ if !ok {+ continue+ }+ masked, err := applyMaskerToData(val, m)+ if err != nil {+ return err+ }+ sortArr[i] = masked+ }+ hitMap["sort"] = sortArr+ return nil+}++// maskElasticsearchSingleHit masks _source, fields, highlight, sort, and inner_hits for a single hit.+func maskElasticsearchSingleHit(hitMap map[string]any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {+ if source, ok := hitMap["_source"].(map[string]any); ok {+ masked, err := maskElasticsearchSourceObject(source, objectSchema, semanticTypeToMasker)+ if err != nil {+ return err+ }+ hitMap["_source"] = masked+ }+ if err := maskElasticsearchHitFields(hitMap, objectSchema, semanticTypeToMasker); err != nil {+ return err+ }+ if err := maskElasticsearchHitHighlight(hitMap, objectSchema, semanticTypeToMasker); err != nil {+ return err+ }+ if err := maskElasticsearchHitSort(hitMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {+ return err+ }+ return maskElasticsearchHitInnerHits(hitMap, sortFields, objectSchema, semanticTypeToMasker)+}++// maskElasticsearchHitInnerHits masks the inner_hits section of a hit.+// inner_hits.<name>.hits.hits[] has the same structure as outer hits,+// so masking is applied recursively using the same ObjectSchema.+func maskElasticsearchHitInnerHits(hitMap map[string]any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {+ innerHitsVal, ok := hitMap["inner_hits"]+ if !ok {+ return nil+ }+ innerHitsMap, ok := innerHitsVal.(map[string]any)+ if !ok {+ return nil+ }+ for name, ihVal := range innerHitsMap {+ if err := maskElasticsearchInnerHitGroup(name, ihVal, sortFields, objectSchema, semanticTypeToMasker); err != nil {+ return err+ }+ }+ return nil+}++// maskElasticsearchInnerHitGroup masks a single named inner_hits group (e.g. inner_hits.comments).+func maskElasticsearchInnerHitGroup(name string, ihVal any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {+ ihObj, ok := ihVal.(map[string]any)+ if !ok {+ return nil+ }+ hitsVal, ok := ihObj["hits"]+ if !ok {+ return nil+ }+ hitsObj, ok := hitsVal.(map[string]any)+ if !ok {+ return nil+ }+ hitsArray, ok := hitsObj["hits"].([]any)+ if !ok {+ return nil+ }+ for i, innerHit := range hitsArray {+ innerHitMap, ok := innerHit.(map[string]any)+ if !ok {+ continue+ }+ if err := maskElasticsearchSingleHit(innerHitMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {+ return errors.Wrapf(err, "failed to mask inner_hits.%s hit %d", name, i)+ }+ hitsArray[i] = innerHitMap+ }+ return nil+}++// maskElasticsearchHitsColumn masks _source, fields, highlight, sort, and inner_hits in each hit+// within a hits column value. The hitsColumnJSON is the JSON string from the "hits" column,+// e.g. {"total":{"value":1},"hits":[{"_source":{...},"fields":{...},"highlight":{...},"sort":[...]}]}.+// sortFields is the ordered list of field names from the request's sort clause.+func maskElasticsearchHitsColumn(hitsColumnJSON string, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {+ var hitsObj map[string]any+ if err := json.Unmarshal([]byte(hitsColumnJSON), &hitsObj); err != nil {+ return "", errors.Wrap(err, "failed to unmarshal hits column")+ }++ hitsArray, ok := hitsObj["hits"].([]any)+ if !ok {+ return hitsColumnJSON, nil+ }++ for i, hit := range hitsArray {+ hitMap, ok := hit.(map[string]any)+ if !ok {+ continue+ }+ if err := maskElasticsearchSingleHit(hitMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {+ return "", errors.Wrapf(err, "failed to mask hit %d", i)+ }+ hitsArray[i] = hitMap+ }+ hitsObj["hits"] = hitsArray++ out, err := json.Marshal(hitsObj)+ if err != nil {+ return "", errors.Wrap(err, "failed to marshal masked hits column")+ }+ return string(out), nil+}++// maskElasticsearchDocSource masks the _source column value for a GET _doc response.+// The sourceColumnJSON is the JSON string from the "_source" column, which is the raw document.+func maskElasticsearchDocSource(sourceColumnJSON string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {+ var source map[string]any+ if err := json.Unmarshal([]byte(sourceColumnJSON), &source); err != nil {+ return "", errors.Wrap(err, "failed to unmarshal _source column")+ }++ masked, err := maskElasticsearchSourceObject(source, objectSchema, semanticTypeToMasker)+ if err != nil {+ return "", err+ }++ out, err := json.Marshal(masked)+ if err != nil {+ return "", errors.Wrap(err, "failed to marshal masked _source column")+ }+ return string(out), nil+}++// maskElasticsearchGetSourceColumn masks a single field value from a GET _source response.+// In _source responses, each top-level document field becomes a separate column.+// fieldName is the column name (= field name), valueJSON is the marshaled value.+func maskElasticsearchGetSourceColumn(fieldName, valueJSON string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {+ var childSchema *storepb.ObjectSchema+ if structKind := objectSchema.GetStructKind(); structKind != nil {+ if props := structKind.GetProperties(); props != nil {+ childSchema = props[fieldName]+ }+ }+ if childSchema == nil {+ return valueJSON, nil+ }++ var value any+ if err := json.Unmarshal([]byte(valueJSON), &value); err != nil {+ return "", errors.Wrapf(err, "failed to unmarshal field %q", fieldName)+ }++ masked, err := walkAndMaskJSONRecursive(value, childSchema, semanticTypeToMasker)+ if err != nil {+ return "", err+ }++ out, err := json.Marshal(masked)+ if err != nil {+ return "", errors.Wrapf(err, "failed to marshal masked field %q", fieldName)+ }+ return string(out), nil+}++// maskElasticsearchMGetSource masks _source in each doc within a _mget response.+// The docsColumnJSON is the JSON string from the "docs" column.+func maskElasticsearchMGetSource(docsColumnJSON string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {+ var docs []any+ if err := json.Unmarshal([]byte(docsColumnJSON), &docs); err != nil {+ return "", errors.Wrap(err, "failed to unmarshal docs column")+ }++ for i, doc := range docs {+ docMap, ok := doc.(map[string]any)+ if !ok {+ continue+ }+ if source, ok := docMap["_source"].(map[string]any); ok {+ masked, err := maskElasticsearchSourceObject(source, objectSchema, semanticTypeToMasker)+ if err != nil {+ return "", errors.Wrapf(err, "failed to mask _source in doc %d", i)+ }+ docMap["_source"] = masked+ }+ docs[i] = docMap+ }++ out, err := json.Marshal(docs)+ if err != nil {+ return "", errors.Wrap(err, "failed to marshal masked docs column")+ }+ return string(out), nil+}++// maskElasticsearchMSearchResponses masks the "responses" column from a _msearch response.+// Each element in the responses array is a full search response with hits.+func maskElasticsearchMSearchResponses(responsesColumnJSON string, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {+ var responses []any+ if err := json.Unmarshal([]byte(responsesColumnJSON), &responses); err != nil {+ return "", errors.Wrap(err, "failed to unmarshal responses column")+ }++ for i, resp := range responses {+ respMap, ok := resp.(map[string]any)+ if !ok {+ continue+ }+ if err := maskElasticsearchMSearchSingleResponse(respMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {+ return "", errors.Wrapf(err, "failed to mask response %d", i)+ }+ }++ out, err := json.Marshal(responses)+ if err != nil {+ return "", errors.Wrap(err, "failed to marshal masked responses column")+ }+ return string(out), nil+}++// maskElasticsearchMSearchSingleResponse masks the hits in a single _msearch response element.+func maskElasticsearchMSearchSingleResponse(respMap map[string]any, sortFields []string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) error {+ hitsVal, ok := respMap["hits"]+ if !ok {+ return nil+ }+ hitsObj, ok := hitsVal.(map[string]any)+ if !ok {+ return nil+ }+ hitsArray, ok := hitsObj["hits"].([]any)+ if !ok {+ return nil+ }+ for j, hit := range hitsArray {+ hitMap, ok := hit.(map[string]any)+ if !ok {+ continue+ }+ if err := maskElasticsearchSingleHit(hitMap, sortFields, objectSchema, semanticTypeToMasker); err != nil {+ return errors.Wrapf(err, "failed to mask hit %d", j)+ }+ hitsArray[j] = hitMap+ }+ return nil+}++// checkElasticsearchRequestBlocked returns an error if the request must be blocked+// because masking policies exist on the target index.+func checkElasticsearchRequestBlocked(analysis *esparser.RequestAnalysis) error {+ if analysis.API == esparser.APIBlocked {+ return errors.New("this Elasticsearch API is not supported when data masking is configured on the target index")+ }+ if len(analysis.BlockedFeatures) > 0 {+ names := make([]string, 0, len(analysis.BlockedFeatures))+ for _, f := range analysis.BlockedFeatures {+ names = append(names, esparser.BlockedFeatureNames[f])+ }+ return errors.Errorf("the following features are not supported when data masking is configured: %s", strings.Join(names, ", "))+ }+ return nil+}++// getElasticsearchIndexObjectSchema retrieves the ObjectSchema for an ES index from the database config.+func getElasticsearchIndexObjectSchema(ctx context.Context, stores *store.Store, instanceID string, databaseName string, indexName string) (*storepb.ObjectSchema, error) {+ dbMetadata, err := stores.GetDBSchema(ctx, &store.FindDBSchemaMessage{+ InstanceID: instanceID,+ DatabaseName: databaseName,+ })+ if err != nil {+ return nil, errors.Wrapf(err, "failed to get database schema: %q", databaseName)+ }++ if dbMetadata == nil {+ return nil, nil+ }++ schemas := dbMetadata.GetConfig().GetSchemas()+ if len(schemas) == 0 {+ return nil, nil+ }++ schema := schemas[0]+ tables := schema.GetTables()+ for _, table := range tables {+ if table.GetName() == indexName {+ return table.GetObjectSchema(), nil+ }+ }++ return nil, nil+}++// objectSchemaHasSemanticTypes recursively checks if any node in the ObjectSchema has a semantic type set.+func objectSchemaHasSemanticTypes(os *storepb.ObjectSchema) bool {+ if os == nil {+ return false+ }+ if os.SemanticType != "" {+ return true+ }+ if sk := os.GetStructKind(); sk != nil {+ for _, prop := range sk.GetProperties() {+ if objectSchemaHasSemanticTypes(prop) {+ return true+ }+ }+ }+ if ak := os.GetArrayKind(); ak != nil {+ if objectSchemaHasSemanticTypes(ak.GetKind()) {+ return true+ }+ }+ return false+}diff --git a/backend/api/v1/catalog_masking_elasticsearch_test.go b/backend/api/v1/catalog_masking_elasticsearch_test.gonew file mode 100644index 0000000000..48ed67e6b8--- /dev/null+++ b/backend/api/v1/catalog_masking_elasticsearch_test.go@@ -0,0 +1,314 @@+package v1++import (+ "encoding/json"+ "io"+ "os"+ "testing"++ "github.com/stretchr/testify/require"+ "gopkg.in/yaml.v3"++ "github.com/bytebase/bytebase/backend/component/masker"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"+)++// testElasticsearchObjectSchema creates a schema where "email" and "contact.phone" are masked.+func testElasticsearchObjectSchema() *storepb.ObjectSchema {+ return &storepb.ObjectSchema{+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "name": {Type: storepb.ObjectSchema_STRING},+ "email": {Type: storepb.ObjectSchema_STRING, SemanticType: "bb.default"},+ "age": {Type: storepb.ObjectSchema_NUMBER},+ "contact": {+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "phone": {Type: storepb.ObjectSchema_STRING, SemanticType: "bb.default"},+ "city": {Type: storepb.ObjectSchema_STRING},+ },+ },+ },+ },+ },+ },+ },+ }+}++func testElasticsearchMaskerMap() map[string]masker.Masker {+ return map[string]masker.Masker{+ "bb.default": masker.NewDefaultFullMasker(),+ }+}++type elasticsearchTestData struct {+ LookupSemanticType []lookupSemanticTypeCase `yaml:"lookupSemanticType"`+ MaskHitsColumn []maskJSONCase `yaml:"maskHitsColumn"`+ MaskDocSource []maskJSONCase `yaml:"maskDocSource"`+ MaskGetSourceColumn []maskGetSourceCase `yaml:"maskGetSourceColumn"`+ MaskMGetSource []maskJSONCase `yaml:"maskMGetSource"`+ MaskMSearchResponses []maskJSONCase `yaml:"maskMSearchResponses"`+ MaskInnerHits []maskJSONCase `yaml:"maskInnerHits"`+ CheckBlocked []checkBlockedCase `yaml:"checkBlocked"`+}++type lookupSemanticTypeCase struct {+ Description string `yaml:"description"`+ DotPath string `yaml:"dotPath"`+ WantType string `yaml:"wantType"`+}++type maskJSONCase struct {+ Description string `yaml:"description"`+ Input string `yaml:"input"`+ Want string `yaml:"want"`+ SortFields []string `yaml:"sortFields"`+}++type maskGetSourceCase struct {+ Description string `yaml:"description"`+ FieldName string `yaml:"fieldName"`+ Input string `yaml:"input"`+ Want string `yaml:"want"`+}++type checkBlockedCase struct {+ Description string `yaml:"description"`+ API esparser.MaskableAPI `yaml:"api"`+ Index string `yaml:"index"`+ BlockedFeatures []esparser.BlockedFeature `yaml:"blockedFeatures"`+ WantError bool `yaml:"wantError"`+ WantErrorContains string `yaml:"wantErrorContains"`+}++func loadElasticsearchTestData(t *testing.T) *elasticsearchTestData {+ t.Helper()+ f, err := os.Open("test-data/elasticsearch_masking.yaml")+ require.NoError(t, err)+ raw, err := io.ReadAll(f)+ require.NoError(t, err)+ require.NoError(t, f.Close())++ var data elasticsearchTestData+ require.NoError(t, yaml.Unmarshal(raw, &data))+ return &data+}++// requireJSONEqual compares two JSON strings by unmarshalling and comparing as Go values.+func requireJSONEqual(t *testing.T, want, got string) {+ t.Helper()+ var wantVal, gotVal any+ require.NoError(t, json.Unmarshal([]byte(want), &wantVal))+ require.NoError(t, json.Unmarshal([]byte(got), &gotVal))+ require.Equal(t, wantVal, gotVal)+}++func TestLookupSemanticTypeByDotPath(t *testing.T) {+ td := loadElasticsearchTestData(t)+ schema := testElasticsearchObjectSchema()++ for _, tc := range td.LookupSemanticType {+ t.Run(tc.Description, func(t *testing.T) {+ got := lookupSemanticTypeByDotPath(tc.DotPath, schema)+ require.Equal(t, tc.WantType, got)+ })+ }++ t.Run("nil schema", func(t *testing.T) {+ got := lookupSemanticTypeByDotPath("email", nil)+ require.Equal(t, "", got)+ })+}++func TestMaskElasticsearchHitsColumn(t *testing.T) {+ td := loadElasticsearchTestData(t)+ schema := testElasticsearchObjectSchema()+ maskers := testElasticsearchMaskerMap()++ for _, tc := range td.MaskHitsColumn {+ t.Run(tc.Description, func(t *testing.T) {+ result, err := maskElasticsearchHitsColumn(tc.Input, tc.SortFields, schema, maskers)+ require.NoError(t, err)+ requireJSONEqual(t, tc.Want, result)+ })+ }+}++func TestMaskElasticsearchDocSource(t *testing.T) {+ td := loadElasticsearchTestData(t)+ schema := testElasticsearchObjectSchema()+ maskers := testElasticsearchMaskerMap()++ for _, tc := range td.MaskDocSource {+ t.Run(tc.Description, func(t *testing.T) {+ result, err := maskElasticsearchDocSource(tc.Input, schema, maskers)+ require.NoError(t, err)+ requireJSONEqual(t, tc.Want, result)+ })+ }+}++func TestMaskElasticsearchGetSourceColumn(t *testing.T) {+ td := loadElasticsearchTestData(t)+ schema := testElasticsearchObjectSchema()+ maskers := testElasticsearchMaskerMap()++ for _, tc := range td.MaskGetSourceColumn {+ t.Run(tc.Description, func(t *testing.T) {+ result, err := maskElasticsearchGetSourceColumn(tc.FieldName, tc.Input, schema, maskers)+ require.NoError(t, err)+ requireJSONEqual(t, tc.Want, result)+ })+ }+}++func TestMaskElasticsearchMGetSource(t *testing.T) {+ td := loadElasticsearchTestData(t)+ schema := testElasticsearchObjectSchema()+ maskers := testElasticsearchMaskerMap()++ for _, tc := range td.MaskMGetSource {+ t.Run(tc.Description, func(t *testing.T) {+ result, err := maskElasticsearchMGetSource(tc.Input, schema, maskers)+ require.NoError(t, err)+ requireJSONEqual(t, tc.Want, result)+ })+ }+}++func TestMaskElasticsearchInnerHits(t *testing.T) {+ td := loadElasticsearchTestData(t)+ schema := testElasticsearchObjectSchema()+ maskers := testElasticsearchMaskerMap()++ for _, tc := range td.MaskInnerHits {+ t.Run(tc.Description, func(t *testing.T) {+ result, err := maskElasticsearchHitsColumn(tc.Input, tc.SortFields, schema, maskers)+ require.NoError(t, err)+ requireJSONEqual(t, tc.Want, result)+ })+ }+}++func TestMaskElasticsearchMSearchResponses(t *testing.T) {+ td := loadElasticsearchTestData(t)+ schema := testElasticsearchObjectSchema()+ maskers := testElasticsearchMaskerMap()++ for _, tc := range td.MaskMSearchResponses {+ t.Run(tc.Description, func(t *testing.T) {+ result, err := maskElasticsearchMSearchResponses(tc.Input, tc.SortFields, schema, maskers)+ require.NoError(t, err)+ requireJSONEqual(t, tc.Want, result)+ })+ }+}++func TestMaskElasticsearchSourceObjectDirectReplacement(t *testing.T) {+ maskers := map[string]masker.Masker{+ "bb.default": masker.NewDefaultFullMasker(),+ }++ tests := []struct {+ description string+ input map[string]any+ schema *storepb.ObjectSchema+ want map[string]any+ }{+ {+ description: "tagged field with nested object is replaced directly",+ input: map[string]any{+ "name": "Alice",+ "contact": map[string]any{+ "phone": "555-1234",+ "city": "NYC",+ },+ },+ schema: &storepb.ObjectSchema{+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "name": {Type: storepb.ObjectSchema_STRING},+ "contact": {+ Type: storepb.ObjectSchema_OBJECT,+ SemanticType: "bb.default",+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "phone": {Type: storepb.ObjectSchema_STRING},+ "city": {Type: storepb.ObjectSchema_STRING},+ },+ },+ },+ },+ },+ },+ },+ },+ want: map[string]any{+ "name": "Alice",+ "contact": "******",+ },+ },+ {+ description: "tagged field with array value is replaced directly",+ input: map[string]any{+ "name": "Alice",+ "tags": []any{"tag1", "tag2"},+ },+ schema: &storepb.ObjectSchema{+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "name": {Type: storepb.ObjectSchema_STRING},+ "tags": {+ Type: storepb.ObjectSchema_ARRAY,+ SemanticType: "bb.default",+ },+ },+ },+ },+ },+ want: map[string]any{+ "name": "Alice",+ "tags": "******",+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.description, func(t *testing.T) {+ got, err := maskElasticsearchSourceObject(tc.input, tc.schema, maskers)+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ })+ }+}++func TestCheckElasticsearchRequestBlocked(t *testing.T) {+ td := loadElasticsearchTestData(t)++ for _, tc := range td.CheckBlocked {+ t.Run(tc.Description, func(t *testing.T) {+ analysis := &esparser.RequestAnalysis{+ API: tc.API,+ Index: tc.Index,+ BlockedFeatures: tc.BlockedFeatures,+ }+ err := checkElasticsearchRequestBlocked(analysis)+ if tc.WantError {+ require.Error(t, err)+ if tc.WantErrorContains != "" {+ require.Contains(t, err.Error(), tc.WantErrorContains)+ }+ } else {+ require.NoError(t, err)+ }+ })+ }+}diff --git a/backend/api/v1/catalog_masking_test.go b/backend/api/v1/catalog_masking_test.goindex dc2cd5bbd7..200e176c00 100644--- a/backend/api/v1/catalog_masking_test.go+++ b/backend/api/v1/catalog_masking_test.go@@ -223,7 +223,7 @@ func TestWalkAndMaskJSON(t *testing.T) {want: `{"name": "John", "information": {"ssn": "******"}}`,},{- description: "Recursive apply the masker to the object",+ description: "tagged root replaces each field value directly",input: `{"name": "John", "information": {"ssn": "123-45-6789"}}`,objectSchema: &storepb.ObjectSchema{Type: storepb.ObjectSchema_OBJECT,@@ -232,7 +232,7 @@ func TestWalkAndMaskJSON(t *testing.T) {semanticTypeToMasker: map[string]masker.Masker{"SSN": masker.NewFullMasker("******"),},- want: `{"name": "******", "information": {"ssn": "******"}}`,+ want: `{"name": "******", "information": "******"}`,},{description: "Mask the array",@@ -262,6 +262,63 @@ func TestWalkAndMaskJSON(t *testing.T) {},want: `{"name": "John", "information": ["******", "******"]}`,},+ {+ description: "tagged field with object value is replaced directly",+ input: `{"name": "John", "address": {"street": "123 Main St", "city": "NYC"}}`,+ objectSchema: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "address": {+ Type: storepb.ObjectSchema_OBJECT,+ SemanticType: "PII",+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "street": {Type: storepb.ObjectSchema_STRING},+ "city": {Type: storepb.ObjectSchema_STRING},+ },+ },+ },+ },+ },+ },+ },+ },+ semanticTypeToMasker: map[string]masker.Masker{+ "PII": masker.NewFullMasker("******"),+ },+ want: `{"name": "John", "address": "******"}`,+ },+ {+ description: "tagged field with array value is replaced directly",+ input: `{"name": "John", "tags": ["tag1", "tag2", "tag3"]}`,+ objectSchema: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "tags": {+ Type: storepb.ObjectSchema_ARRAY,+ SemanticType: "PII",+ Kind: &storepb.ObjectSchema_ArrayKind_{+ ArrayKind: &storepb.ObjectSchema_ArrayKind{+ Kind: &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_STRING,+ },+ },+ },+ },+ },+ },+ },+ },+ semanticTypeToMasker: map[string]masker.Masker{+ "PII": masker.NewFullMasker("******"),+ },+ want: `{"name": "John", "tags": "******"}`,+ },{description: "mask following the field paths",input: `{@@ -314,9 +371,7 @@ func TestWalkAndMaskJSON(t *testing.T) {},want: `{"firstName": "John",- "address": {- "street": "******"- },+ "address": "******","creditCard": {"number": "******"}diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex 3da26a4ba5..5050b9fba7 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -32,6 +32,7 @@ import ("github.com/bytebase/bytebase/backend/plugin/advisor/code""github.com/bytebase/bytebase/backend/plugin/db"parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"+ esparser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch""github.com/bytebase/bytebase/backend/plugin/schema""github.com/bytebase/bytebase/backend/runner/schemasync""github.com/bytebase/bytebase/backend/store"@@ -584,6 +585,45 @@ func queryRetry(}}+ // Pre-execution check for Elasticsearch: reject blocked APIs and predicate violations+ // before sending the query to the server.+ if !queryContext.Explain && !queryContext.SkipMasking &&+ instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH &&+ licenseService.IsFeatureEnabledForInstance(v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil {+ for _, stmt := range statements {+ parsed, parseErr := esparser.ParseElasticsearchREST(stmt.Text)+ if parseErr != nil || len(parsed.Requests) == 0 {+ continue+ }+ req := parsed.Requests[0]+ analysis := esparser.AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))+ if analysis.API == esparser.APIUnsupported {+ continue+ }+ indexName := analysis.Index+ if indexName == "" {+ continue+ }+ objectSchema, schemaErr := getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, indexName)+ if schemaErr != nil {+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.Wrapf(schemaErr, "failed to get object schema for index %q", indexName))+ }+ if objectSchema == nil || !objectSchemaHasSemanticTypes(objectSchema) {+ continue+ }+ if err := checkElasticsearchRequestBlocked(analysis); err != nil {+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument, err)+ }+ for _, field := range analysis.PredicateFields {+ semanticType := lookupSemanticTypeByDotPath(field, objectSchema)+ if semanticType != "" {+ return nil, nil, time.Duration(0), connect.NewError(connect.CodeInvalidArgument,+ errors.Errorf("using field %q tagged by semantic type %q in query predicate is not allowed", field, semanticType))+ }+ }+ }+ }+slog.Debug("start execute with timeout", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName), slog.String("statement", originalStatement))results, duration, queryErr := executeWithTimeout(ctx,@@ -730,6 +770,90 @@ func queryRetry(}}}+ } else if instance.Metadata.GetEngine() == storepb.Engine_ELASTICSEARCH {+ for _, result := range results {+ parsed, err := esparser.ParseElasticsearchREST(result.Statement)+ if err != nil || len(parsed.Requests) == 0 {+ continue+ }+ req := parsed.Requests[0]+ analysis := esparser.AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))++ if analysis.API == esparser.APIUnsupported || analysis.API == esparser.APIBlocked {+ continue+ }++ indexName := analysis.Index+ if indexName == "" {+ continue+ }+ objectSchema, err := getElasticsearchIndexObjectSchema(ctx, stores, database.InstanceID, database.DatabaseName, indexName)+ if err != nil {+ return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get object schema for index %q", indexName))+ }+ if objectSchema == nil || !objectSchemaHasSemanticTypes(objectSchema) {+ continue+ }++ semanticTypeToMaskerMap, err := buildSemanticTypeToMaskerMap(ctx, stores)+ if err != nil {+ return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.New(err.Error()))+ }++ // Apply masking based on API type.+ for _, row := range result.Rows {+ for colIdx, colName := range result.ColumnNames {+ if colIdx >= len(row.Values) {+ break+ }+ value := row.Values[colIdx].GetStringValue()+ if value == "" {+ continue+ }++ var maskedValue string+ var maskErr error++ switch analysis.API {+ case esparser.APIMaskSearch:+ switch colName {+ case "hits":+ maskedValue, maskErr = maskElasticsearchHitsColumn(value, analysis.SortFields, objectSchema, semanticTypeToMaskerMap)+ case "responses":+ maskedValue, maskErr = maskElasticsearchMSearchResponses(value, analysis.SortFields, objectSchema, semanticTypeToMaskerMap)+ default:+ continue+ }+ case esparser.APIMaskGetDoc, esparser.APIMaskExplain:+ if colName == "_source" {+ maskedValue, maskErr = maskElasticsearchDocSource(value, objectSchema, semanticTypeToMaskerMap)+ } else {+ continue+ }+ case esparser.APIMaskGetSource:+ maskedValue, maskErr = maskElasticsearchGetSourceColumn(colName, value, objectSchema, semanticTypeToMaskerMap)+ case esparser.APIMaskMGet:+ if colName == "docs" {+ maskedValue, maskErr = maskElasticsearchMGetSource(value, objectSchema, semanticTypeToMaskerMap)+ } else {+ continue+ }+ default:+ continue+ }++ if maskErr != nil {+ return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.Wrapf(maskErr, "failed to mask ES response"))+ }++ row.Values[colIdx] = &v1pb.RowValue{+ Kind: &v1pb.RowValue_StringValue{+ StringValue: maskedValue,+ },+ }+ }+ }+ }} else {masker := NewQueryResultMasker(stores)if err := masker.MaskResults(ctx, spans, results, instance, user); err != nil {diff --git a/backend/api/v1/test-data/elasticsearch_masking.yaml b/backend/api/v1/test-data/elasticsearch_masking.yamlnew file mode 100644index 0000000000..d2dc92f946--- /dev/null+++ b/backend/api/v1/test-data/elasticsearch_masking.yaml@@ -0,0 +1,169 @@+# Test data for Elasticsearch response masking.+#+# ObjectSchema used by all tests:+# - "email" : STRING, semanticType "bb.default"+# - "contact.phone" : STRING, semanticType "bb.default"+# - "name" : STRING (no semantic type)+# - "age" : NUMBER (no semantic type)+# - "contact.city" : STRING (no semantic type)+#+# The masker for "bb.default" is DefaultFullMasker which replaces values with "******".+#+# MaskableAPI enum: APIUnsupported=0, APIMaskSearch=1, APIMaskGetDoc=2, APIMaskGetSource=3, APIMaskMGet=4, APIMaskExplain=5, APIBlocked=6+# BlockedFeature enum: BlockedFeatureAggs=0, BlockedFeatureSuggest=1, BlockedFeatureScriptFields=2,+# BlockedFeatureRuntimeMappings=3, BlockedFeatureStoredFields=4, BlockedFeatureDocvalueFields=5++lookupSemanticType:+ - description: "email field"+ dotPath: "email"+ wantType: "bb.default"+ - description: "nested contact.phone"+ dotPath: "contact.phone"+ wantType: "bb.default"+ - description: "name field (no semantic type)"+ dotPath: "name"+ wantType: ""+ - description: "age field (no semantic type)"+ dotPath: "age"+ wantType: ""+ - description: "contact.city (no semantic type)"+ dotPath: "contact.city"+ wantType: ""+ - description: "unknown field"+ dotPath: "unknown"+ wantType: ""+ - description: "contact.unknown"+ dotPath: "contact.unknown"+ wantType: ""+ - description: "deeply.nested.path"+ dotPath: "deeply.nested.path"+ wantType: ""+ - description: "empty path"+ dotPath: ""+ wantType: ""++maskHitsColumn:+ - description: "masks _source fields"+ input: '{"total":{"value":1},"hits":[{"_index":"test","_id":"1","_source":{"name":"Alice","email":"[email protected]","contact":{"phone":"555-1234","city":"NYC"}}}]}'+ want: '{"total":{"value":1},"hits":[{"_index":"test","_id":"1","_source":{"name":"Alice","email":"******","contact":{"phone":"******","city":"NYC"}}}]}'+ - description: "no hits array"+ input: '{"total":{"value":0}}'+ want: '{"total":{"value":0}}'+ - description: "hit without _source"+ input: '{"total":{"value":1},"hits":[{"_index":"test","_id":"1"}]}'+ want: '{"total":{"value":1},"hits":[{"_index":"test","_id":"1"}]}'+ - description: "multiple hits"+ input: '{"total":{"value":2},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"}},{"_id":"2","_source":{"name":"Bob","email":"[email protected]"}}]}'+ want: '{"total":{"value":2},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"}},{"_id":"2","_source":{"name":"Bob","email":"******"}}]}'+ - description: "masks fields section"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"fields":{"email":["[email protected]"],"contact.phone":["555-1234"],"name":["Alice"]}}]}'+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"fields":{"email":["******"],"contact.phone":["******"],"name":["Alice"]}}]}'+ - description: "masks highlight section"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"highlight":{"email":["<em>alice</em>@example.com"],"name":["<em>Alice</em>"]}}]}'+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"highlight":{"email":["******"],"name":["<em>Alice</em>"]}}]}'+ - description: "masks sort array"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"sort":[1234567890,"[email protected]"]}]}'+ sortFields:+ - "age"+ - "email"+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"sort":[1234567890,"******"]}]}'+ - description: "masks fields highlight and sort together"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"fields":{"email":["[email protected]"],"contact.phone":["555-1234"],"name":["Alice"]},"highlight":{"email":["<em>alice</em>@example.com"],"name":["<em>Alice</em>"]},"sort":[1234567890,"[email protected]"]}]}'+ sortFields:+ - "age"+ - "email"+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"fields":{"email":["******"],"contact.phone":["******"],"name":["Alice"]},"highlight":{"email":["******"],"name":["<em>Alice</em>"]},"sort":[1234567890,"******"]}]}'+ - description: "sort with fewer sortFields than sort values"+ input: '{"total":{"value":1},"hits":[{"_id":"1","sort":[1234567890,"[email protected]","extra"]}]}'+ sortFields:+ - "age"+ want: '{"total":{"value":1},"hits":[{"_id":"1","sort":[1234567890,"[email protected]","extra"]}]}'++maskDocSource:+ - description: "masks _source column"+ input: '{"name":"Alice","email":"[email protected]","contact":{"phone":"555-1234","city":"NYC"}}'+ want: '{"name":"Alice","email":"******","contact":{"phone":"******","city":"NYC"}}'++maskGetSourceColumn:+ - description: "masks sensitive field"+ fieldName: "email"+ input: '"[email protected]"'+ want: '"******"'+ - description: "does not mask non-sensitive field"+ fieldName: "name"+ input: '"Alice"'+ want: '"Alice"'+ - description: "masks nested object field"+ fieldName: "contact"+ input: '{"phone":"555-1234","city":"NYC"}'+ want: '{"phone":"******","city":"NYC"}'+ - description: "unknown field passes through"+ fieldName: "unknown_field"+ input: '"value"'+ want: '"value"'++maskMGetSource:+ - description: "masks docs"+ input: '[{"_index":"test","_id":"1","found":true,"_source":{"name":"Alice","email":"[email protected]"}},{"_index":"test","_id":"2","found":true,"_source":{"name":"Bob","email":"[email protected]"}}]'+ want: '[{"_index":"test","_id":"1","found":true,"_source":{"name":"Alice","email":"******"}},{"_index":"test","_id":"2","found":true,"_source":{"name":"Bob","email":"******"}}]'++maskMSearchResponses:+ - description: "masks each response"+ input: '[{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"}}]}},{"hits":{"total":{"value":1},"hits":[{"_id":"2","_source":{"name":"Bob","email":"[email protected]","contact":{"phone":"555-1234","city":"NYC"}}}]}}]'+ want: '[{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"}}]}},{"hits":{"total":{"value":1},"hits":[{"_id":"2","_source":{"name":"Bob","email":"******","contact":{"phone":"******","city":"NYC"}}}]}}]'+ - description: "empty responses array"+ input: '[]'+ want: '[]'+ - description: "response without hits passes through"+ input: '[{"status":200},{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"}}]}}]'+ want: '[{"status":200},{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"}}]}}]'+ - description: "response with inner_hits"+ input: '[{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"inner_hits":{"comments":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Bob","email":"[email protected]"}}]}}}}]}}]'+ want: '[{"hits":{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"inner_hits":{"comments":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Bob","email":"******"}}]}}}}]}}]'++maskInnerHits:+ - description: "masks inner_hits _source"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"inner_hits":{"comments":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Bob","email":"[email protected]","contact":{"phone":"555-9999","city":"LA"}}}]}}}}]}'+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"inner_hits":{"comments":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Bob","email":"******","contact":{"phone":"******","city":"LA"}}}]}}}}]}'+ - description: "masks inner_hits fields and highlight"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"inner_hits":{"nested":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Inner","email":"[email protected]"},"fields":{"email":["[email protected]"],"name":["Inner"]},"highlight":{"email":["<em>inner</em>@example.com"]}}]}}}}]}'+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"inner_hits":{"nested":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"Inner","email":"******"},"fields":{"email":["******"],"name":["Inner"]},"highlight":{"email":["******"]}}]}}}}]}'+ - description: "masks nested inner_hits recursively"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Root","email":"[email protected]"},"inner_hits":{"level1":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"L1","email":"[email protected]"},"inner_hits":{"level2":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"L2","email":"[email protected]"}}]}}}}]}}}}]}'+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Root","email":"******"},"inner_hits":{"level1":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"L1","email":"******"},"inner_hits":{"level2":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"L2","email":"******"}}]}}}}]}}}}]}'+ - description: "no inner_hits passes through"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"}}]}'+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"}}]}'+ - description: "multiple inner_hits groups"+ input: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"[email protected]"},"inner_hits":{"group_a":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"GA","email":"[email protected]"}}]}},"group_b":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"GB","email":"[email protected]"}}]}}}}]}'+ want: '{"total":{"value":1},"hits":[{"_id":"1","_source":{"name":"Alice","email":"******"},"inner_hits":{"group_a":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"GA","email":"******"}}]}},"group_b":{"hits":{"total":{"value":1},"hits":[{"_source":{"name":"GB","email":"******"}}]}}}}]}'++checkBlocked:+ - description: "allowed search"+ api: 1+ index: "my-index"+ wantError: false+ - description: "blocked API"+ api: 6+ index: "my-index"+ wantError: true+ wantErrorContains: "not supported when data masking"+ - description: "blocked feature aggs"+ api: 1+ index: "my-index"+ blockedFeatures:+ - 0+ wantError: true+ wantErrorContains: "aggregations"+ - description: "multiple blocked features"+ api: 1+ index: "my-index"+ blockedFeatures:+ - 0+ - 1+ - 2+ wantError: true+ wantErrorContains: "aggregations"+ - description: "unsupported API passes through"+ api: 0+ wantError: falsediff --git a/backend/plugin/parser/elasticsearch/masking.go b/backend/plugin/parser/elasticsearch/masking.gonew file mode 100644index 0000000000..70d5fef3ea--- /dev/null+++ b/backend/plugin/parser/elasticsearch/masking.go@@ -0,0 +1,493 @@+package elasticsearch++import (+ "encoding/json"+ "slices"+ "strings"+)++// MaskableAPI categorizes ES endpoints for masking decisions.+type MaskableAPI int++const (+ // APIUnsupported means the endpoint is not relevant to masking (no user data).+ APIUnsupported MaskableAPI = iota+ // APIMaskSearch covers _search and _msearch.+ APIMaskSearch+ // APIMaskGetDoc covers GET /<index>/_doc/<id>.+ APIMaskGetDoc+ // APIMaskGetSource covers GET /<index>/_source/<id>.+ APIMaskGetSource+ // APIMaskMGet covers _mget.+ APIMaskMGet+ // APIMaskExplain covers _explain.+ APIMaskExplain+ // APIBlocked means the endpoint must be rejected when masking is active.+ APIBlocked+)++// BlockedFeature identifies a feature in the request body that must be blocked.+type BlockedFeature int++const (+ BlockedFeatureAggs BlockedFeature = iota+ BlockedFeatureSuggest+ BlockedFeatureScriptFields+ BlockedFeatureRuntimeMappings+ BlockedFeatureStoredFields+ BlockedFeatureDocvalueFields+)++// BlockedFeatureNames maps BlockedFeature to human-readable names for error messages.+var BlockedFeatureNames = map[BlockedFeature]string{+ BlockedFeatureAggs: "aggregations",+ BlockedFeatureSuggest: "suggest",+ BlockedFeatureScriptFields: "script_fields",+ BlockedFeatureRuntimeMappings: "runtime_mappings",+ BlockedFeatureStoredFields: "stored_fields",+ BlockedFeatureDocvalueFields: "docvalue_fields",+}++// RequestAnalysis is the result of analyzing an ES REST request for masking.+type RequestAnalysis struct {+ // API is the classification of the endpoint.+ API MaskableAPI+ // Index is the target index name extracted from the URL (empty if not applicable).+ Index string+ // BlockedFeatures lists features found in the request body that must be blocked.+ BlockedFeatures []BlockedFeature+ // SourceFields is the list of fields requested via _source filtering (nil = all fields).+ // An empty slice means _source is disabled.+ SourceFields []string+ // SourceDisabled is true when _source: false is set.+ SourceDisabled bool+ // RequestedFields is the list of fields requested via the "fields" parameter.+ RequestedFields []string+ // HighlightFields is the list of fields requested via "highlight.fields".+ HighlightFields []string+ // SortFields is the list of field paths used in "sort".+ SortFields []string+ // HasInnerHits is true if the query contains inner_hits.+ HasInnerHits bool+ // PredicateFields is the list of field paths used in query predicates.+ PredicateFields []string+}++// blockedEndpoints lists URL patterns that must be rejected when masking is active.+// These overlap with read patterns but cannot be safely masked.+// Includes both Elasticsearch and OpenSearch equivalents.+var blockedEndpoints = []string{+ // Elasticsearch endpoints.+ "_async_search",+ "_search/scroll",+ "_search_template",+ "_msearch/template",+ "_sql",+ "_eql/search",+ "_esql/query",+ "_terms_enum",+ "_termvectors",+ "_mtermvectors",+ "_knn_search",+ // OpenSearch equivalents (plugin-based paths).+ "_plugins/_asynchronous_search",+ "_plugins/_sql",+ "_plugins/_ppl",+}++// classifyMaskableAPI determines the MaskableAPI type and target index for an ES request.+func classifyMaskableAPI(method, url string) (MaskableAPI, string) {+ method = strings.ToUpper(method)++ // Strip query parameters.+ path := url+ if i := strings.IndexByte(path, '?'); i >= 0 {+ path = path[:i]+ }++ // Strip leading slash.+ path = strings.TrimPrefix(path, "/")++ // Only GET and POST can return user data.+ switch method {+ case "GET":+ if blocked, index := isBlockedEndpoint(path); blocked {+ return APIBlocked, index+ }+ return classifyGetForMasking(path)+ case "POST":+ if blocked, index := isBlockedEndpoint(path); blocked {+ return APIBlocked, index+ }+ return classifyPostForMasking(path)+ default:+ // HEAD, PUT, DELETE, PATCH do not return user data.+ return APIUnsupported, ""+ }+}++// isBlockedEndpoint checks whether the path matches a blocked endpoint pattern.+// Returns true and the extracted index if blocked.+func isBlockedEndpoint(path string) (bool, string) {+ for _, pattern := range blockedEndpoints {+ if strings.Contains(path, pattern) {+ return true, extractIndex(path)+ }+ }+ return false, ""+}++// classifyGetForMasking classifies a GET request path for masking.+func classifyGetForMasking(path string) (MaskableAPI, string) {+ for part := range strings.SplitSeq(path, "/") {+ switch part {+ case "_search", "_msearch":+ return APIMaskSearch, extractIndex(path)+ case "_doc":+ return APIMaskGetDoc, extractIndex(path)+ case "_source":+ return APIMaskGetSource, extractIndex(path)+ case "_mget":+ return APIMaskMGet, extractIndex(path)+ }+ }++ return APIUnsupported, ""+}++// classifyPostForMasking classifies a POST request path for masking.+func classifyPostForMasking(path string) (MaskableAPI, string) {+ for part := range strings.SplitSeq(path, "/") {+ switch part {+ case "_search", "_msearch":+ return APIMaskSearch, extractIndex(path)+ case "_mget":+ return APIMaskMGet, extractIndex(path)+ case "_explain":+ return APIMaskExplain, extractIndex(path)+ }+ }++ return APIUnsupported, ""+}++// extractIndex returns the first path segment if it does not start with "_".+// This extracts the index name from ES URL paths like "<index>/_search".+func extractIndex(path string) string {+ if path == "" {+ return ""+ }+ parts := strings.SplitN(path, "/", 2)+ if len(parts) == 0 {+ return ""+ }+ first := parts[0]+ if strings.HasPrefix(first, "_") {+ return ""+ }+ return first+}++// AnalyzeRequest combines URL classification with body analysis for an ES request.+// Body analysis is only performed for search and explain APIs.+func AnalyzeRequest(method, url, body string) *RequestAnalysis {+ api, index := classifyMaskableAPI(method, url)+ result := &RequestAnalysis{+ API: api,+ Index: index,+ }++ // Only analyze the body for APIs that accept a query body.+ switch api {+ case APIMaskSearch, APIMaskExplain:+ bodyResult := analyzeRequestBody(body)+ result.BlockedFeatures = bodyResult.BlockedFeatures+ result.SourceFields = bodyResult.SourceFields+ result.SourceDisabled = bodyResult.SourceDisabled+ result.RequestedFields = bodyResult.RequestedFields+ result.HighlightFields = bodyResult.HighlightFields+ result.SortFields = bodyResult.SortFields+ result.HasInnerHits = bodyResult.HasInnerHits+ result.PredicateFields = bodyResult.PredicateFields+ default:+ // No body analysis for other API types.+ }++ return result+}++// analyzeRequestBody parses the JSON body and extracts fields and blocked features.+func analyzeRequestBody(body string) *RequestAnalysis {+ result := &RequestAnalysis{}++ parsed := make(map[string]any)+ if json.Unmarshal([]byte(body), &parsed) != nil {+ return result+ }++ result.BlockedFeatures = detectBlockedFeatures(parsed)+ extractSource(parsed, result)+ result.RequestedFields = extractStringArray(parsed, "fields")+ result.HighlightFields = extractHighlightFields(parsed)+ result.SortFields = extractSortFields(parsed)+ result.HasInnerHits = containsKey(parsed, "inner_hits")+ result.PredicateFields = extractPredicateFields(parsed)++ return result+}++// detectBlockedFeatures checks for top-level keys that indicate blocked features.+func detectBlockedFeatures(parsed map[string]any) []BlockedFeature {+ var blocked []BlockedFeature++ if _, ok := parsed["aggs"]; ok {+ blocked = append(blocked, BlockedFeatureAggs)+ } else if _, ok := parsed["aggregations"]; ok {+ blocked = append(blocked, BlockedFeatureAggs)+ }+ if _, ok := parsed["suggest"]; ok {+ blocked = append(blocked, BlockedFeatureSuggest)+ }+ if _, ok := parsed["script_fields"]; ok {+ blocked = append(blocked, BlockedFeatureScriptFields)+ }+ if _, ok := parsed["runtime_mappings"]; ok {+ blocked = append(blocked, BlockedFeatureRuntimeMappings)+ }+ if _, ok := parsed["stored_fields"]; ok {+ blocked = append(blocked, BlockedFeatureStoredFields)+ }+ if _, ok := parsed["docvalue_fields"]; ok {+ blocked = append(blocked, BlockedFeatureDocvalueFields)+ }++ return blocked+}++// extractSource handles the three forms of _source in the request body.+func extractSource(parsed map[string]any, result *RequestAnalysis) {+ src, ok := parsed["_source"]+ if !ok {+ return+ }++ switch v := src.(type) {+ case bool:+ if !v {+ result.SourceDisabled = true+ }+ case []any:+ result.SourceFields = toStringSlice(v)+ case map[string]any:+ if includes, ok := v["includes"]; ok {+ if arr, ok := includes.([]any); ok {+ result.SourceFields = toStringSlice(arr)+ }+ }+ }+}++// extractStringArray extracts a string array value from a top-level key.+func extractStringArray(parsed map[string]any, key string) []string {+ val, ok := parsed[key]+ if !ok {+ return nil+ }+ arr, ok := val.([]any)+ if !ok {+ return nil+ }+ return toStringSlice(arr)+}++// extractHighlightFields extracts field names from highlight.fields.+func extractHighlightFields(parsed map[string]any) []string {+ highlight, ok := parsed["highlight"]+ if !ok {+ return nil+ }+ hlMap, ok := highlight.(map[string]any)+ if !ok {+ return nil+ }+ fields, ok := hlMap["fields"]+ if !ok {+ return nil+ }+ fieldsMap, ok := fields.(map[string]any)+ if !ok {+ return nil+ }+ result := make([]string, 0, len(fieldsMap))+ for k := range fieldsMap {+ result = append(result, k)+ }+ slices.Sort(result)+ return result+}++// extractSortFields extracts field names from the sort array.+func extractSortFields(parsed map[string]any) []string {+ sortVal, ok := parsed["sort"]+ if !ok {+ return nil+ }+ arr, ok := sortVal.([]any)+ if !ok {+ return nil+ }+ var fields []string+ for _, item := range arr {+ switch v := item.(type) {+ case string:+ if strings.HasPrefix(v, "_") {+ continue+ }+ fields = append(fields, v)+ case map[string]any:+ for k := range v {+ if !strings.HasPrefix(k, "_") {+ fields = append(fields, k)+ }+ }+ }+ }+ return fields+}++// containsKey recursively searches for a key in a nested map structure.+func containsKey(data map[string]any, key string) bool {+ for k, v := range data {+ if k == key {+ return true+ }+ if containsKeyInValue(v, key) {+ return true+ }+ }+ return false+}++// containsKeyInValue checks if any nested map within v contains the given key.+func containsKeyInValue(v any, key string) bool {+ switch child := v.(type) {+ case map[string]any:+ return containsKey(child, key)+ case []any:+ for _, item := range child {+ if m, ok := item.(map[string]any); ok && containsKey(m, key) {+ return true+ }+ }+ }+ return false+}++// toStringSlice converts a []any to a []string, skipping non-string elements.+func toStringSlice(arr []any) []string {+ result := make([]string, 0, len(arr))+ for _, item := range arr {+ if s, ok := item.(string); ok {+ result = append(result, s)+ }+ }+ return result+}++// directFieldClauses are clause types where the keys inside the clause object are field names.+var directFieldClauses = map[string]bool{+ "match": true, "match_phrase": true, "match_phrase_prefix": true, "match_bool_prefix": true,+ "term": true, "terms": true, "terms_set": true,+ "range": true,+ "wildcard": true, "prefix": true, "fuzzy": true, "regexp": true,+ "geo_distance": true, "geo_bounding_box": true, "geo_shape": true, "geo_polygon": true,+ "span_term": true,+}++// compoundClauses maps clause types to the keys that contain sub-queries.+var compoundClauses = map[string][]string{+ "bool": {"must", "must_not", "should", "filter"},+ "nested": {"query"},+ "has_child": {"query"},+ "has_parent": {"query"},+ "dis_max": {"queries"},+ "constant_score": {"filter"},+ "boosting": {"positive", "negative"},+ "function_score": {"query"},+}++// extractPredicateFields extracts field names from ES query clauses.+func extractPredicateFields(parsed map[string]any) []string {+ queryVal, ok := parsed["query"]+ if !ok {+ return nil+ }+ queryMap, ok := queryVal.(map[string]any)+ if !ok {+ return nil+ }+ var fields []string+ extractFieldsFromQuery(queryMap, &fields)+ return fields+}++// extractFieldsFromQuery recursively walks ES query clauses to collect field names.+func extractFieldsFromQuery(queryMap map[string]any, fields *[]string) {+ for clauseType, clauseVal := range queryMap {+ clauseObj, ok := clauseVal.(map[string]any)+ if !ok {+ extractFieldsFromQueryArray(clauseVal, fields)+ continue+ }+ switch {+ case directFieldClauses[clauseType]:+ extractDirectFieldNames(clauseObj, fields)+ case clauseType == "exists":+ if fieldVal, ok := clauseObj["field"].(string); ok {+ *fields = append(*fields, fieldVal)+ }+ case compoundClauses[clauseType] != nil:+ extractCompoundClauseFields(clauseObj, compoundClauses[clauseType], fields)+ default:+ extractFieldsFromQuery(clauseObj, fields)+ }+ }+}++// extractFieldsFromQueryArray handles the case where a clause value is an array of query maps.+func extractFieldsFromQueryArray(val any, fields *[]string) {+ arr, ok := val.([]any)+ if !ok {+ return+ }+ for _, item := range arr {+ if m, ok := item.(map[string]any); ok {+ extractFieldsFromQuery(m, fields)+ }+ }+}++// extractDirectFieldNames extracts field names from a direct field clause object.+func extractDirectFieldNames(clauseObj map[string]any, fields *[]string) {+ for fieldName := range clauseObj {+ if !strings.HasPrefix(fieldName, "_") && fieldName != "boost" {+ *fields = append(*fields, fieldName)+ }+ }+}++// extractCompoundClauseFields recurses into sub-query keys of a compound clause.+func extractCompoundClauseFields(clauseObj map[string]any, subQueryKeys []string, fields *[]string) {+ for _, sqKey := range subQueryKeys {+ sqVal, ok := clauseObj[sqKey]+ if !ok {+ continue+ }+ switch sq := sqVal.(type) {+ case map[string]any:+ extractFieldsFromQuery(sq, fields)+ case []any:+ extractFieldsFromQueryArray(sq, fields)+ }+ }+}diff --git a/backend/plugin/parser/elasticsearch/masking_test.go b/backend/plugin/parser/elasticsearch/masking_test.gonew file mode 100644index 0000000000..3a0ee219d3--- /dev/null+++ b/backend/plugin/parser/elasticsearch/masking_test.go@@ -0,0 +1,157 @@+package elasticsearch++import (+ "io"+ "os"+ "testing"++ "github.com/stretchr/testify/require"+ "gopkg.in/yaml.v3"+)++func TestClassifyMaskableAPI(t *testing.T) {+ type testCase struct {+ Description string `yaml:"description"`+ Method string `yaml:"method"`+ URL string `yaml:"url"`+ WantAPI MaskableAPI `yaml:"wantAPI"`+ WantIndex string `yaml:"wantIndex"`+ }++ a := require.New(t)+ data, err := os.Open("test-data/masking_classify_api.yaml")+ a.NoError(err)+ raw, err := io.ReadAll(data)+ a.NoError(err)+ a.NoError(data.Close())++ var cases []testCase+ a.NoError(yaml.Unmarshal(raw, &cases))++ for _, tc := range cases {+ t.Run(tc.Description, func(t *testing.T) {+ api, index := classifyMaskableAPI(tc.Method, tc.URL)+ require.Equal(t, tc.WantAPI, api, "API classification")+ require.Equal(t, tc.WantIndex, index, "index extraction")+ })+ }+}++func TestAnalyzeRequestBody(t *testing.T) {+ type testCase struct {+ Description string `yaml:"description"`+ Body string `yaml:"body"`+ WantBlocked []BlockedFeature `yaml:"wantBlocked"`+ WantSourceDisabled bool `yaml:"wantSourceDisabled"`+ WantSourceFields []string `yaml:"wantSourceFields"`+ WantFields []string `yaml:"wantFields"`+ WantHighlight []string `yaml:"wantHighlight"`+ WantSort []string `yaml:"wantSort"`+ WantInnerHits bool `yaml:"wantInnerHits"`+ }++ a := require.New(t)+ data, err := os.Open("test-data/masking_analyze_body.yaml")+ a.NoError(err)+ raw, err := io.ReadAll(data)+ a.NoError(err)+ a.NoError(data.Close())++ var cases []testCase+ a.NoError(yaml.Unmarshal(raw, &cases))++ for _, tc := range cases {+ t.Run(tc.Description, func(t *testing.T) {+ result := analyzeRequestBody(tc.Body)+ if tc.WantBlocked != nil {+ require.Equal(t, tc.WantBlocked, result.BlockedFeatures)+ } else {+ require.Empty(t, result.BlockedFeatures)+ }+ require.Equal(t, tc.WantSourceDisabled, result.SourceDisabled)+ if tc.WantSourceFields != nil {+ require.Equal(t, tc.WantSourceFields, result.SourceFields)+ }+ if tc.WantFields != nil {+ require.Equal(t, tc.WantFields, result.RequestedFields)+ }+ if tc.WantHighlight != nil {+ require.ElementsMatch(t, tc.WantHighlight, result.HighlightFields)+ }+ if tc.WantSort != nil {+ require.Equal(t, tc.WantSort, result.SortFields)+ }+ require.Equal(t, tc.WantInnerHits, result.HasInnerHits)+ })+ }+}++func TestAnalyzeRequest(t *testing.T) {+ type testCase struct {+ Description string `yaml:"description"`+ Method string `yaml:"method"`+ URL string `yaml:"url"`+ Body string `yaml:"body"`+ WantAPI MaskableAPI `yaml:"wantAPI"`+ WantIndex string `yaml:"wantIndex"`+ WantBlocked []BlockedFeature `yaml:"wantBlocked"`+ WantPredicateFields []string `yaml:"wantPredicateFields"`+ }++ a := require.New(t)+ data, err := os.Open("test-data/masking_analyze_request.yaml")+ a.NoError(err)+ raw, err := io.ReadAll(data)+ a.NoError(err)+ a.NoError(data.Close())++ var cases []testCase+ a.NoError(yaml.Unmarshal(raw, &cases))++ for _, tc := range cases {+ t.Run(tc.Description, func(t *testing.T) {+ result := AnalyzeRequest(tc.Method, tc.URL, tc.Body)+ require.Equal(t, tc.WantAPI, result.API)+ if tc.WantIndex != "" {+ require.Equal(t, tc.WantIndex, result.Index)+ }+ if tc.WantBlocked != nil {+ require.Equal(t, tc.WantBlocked, result.BlockedFeatures)+ } else {+ require.Empty(t, result.BlockedFeatures)+ }+ if tc.WantPredicateFields != nil {+ require.ElementsMatch(t, tc.WantPredicateFields, result.PredicateFields)+ }+ })+ }+}++func TestExtractPredicateFields(t *testing.T) {+ type testCase struct {+ Description string `yaml:"description"`+ Body string `yaml:"body"`+ WantFields []string `yaml:"wantFields"`+ }++ a := require.New(t)+ data, err := os.Open("test-data/masking_predicate_fields.yaml")+ a.NoError(err)+ raw, err := io.ReadAll(data)+ a.NoError(err)+ a.NoError(data.Close())++ var cases []testCase+ a.NoError(yaml.Unmarshal(raw, &cases))++ for _, tc := range cases {+ t.Run(tc.Description, func(t *testing.T) {+ result := analyzeRequestBody(tc.Body)+ if len(tc.WantFields) > 0 {+ require.ElementsMatch(t, tc.WantFields, result.PredicateFields)+ } else {+ require.Empty(t, result.PredicateFields)+ }+ })+ }+}diff --git a/backend/plugin/parser/elasticsearch/test-data/masking_analyze_body.yaml b/backend/plugin/parser/elasticsearch/test-data/masking_analyze_body.yamlnew file mode 100644index 0000000000..89a58d1e78--- /dev/null+++ b/backend/plugin/parser/elasticsearch/test-data/masking_analyze_body.yaml@@ -0,0 +1,100 @@+# Test cases for analyzeRequestBody.+# BlockedFeature enum: BlockedFeatureAggs=0, BlockedFeatureSuggest=1, BlockedFeatureScriptFields=2,+# BlockedFeatureRuntimeMappings=3, BlockedFeatureStoredFields=4, BlockedFeatureDocvalueFields=5+- description: "empty body"+ body: "{}"+ wantSourceDisabled: false+ wantInnerHits: false+- description: "simple query only"+ body: '{"query": {"match_all": {}}}'+ wantSourceDisabled: false+ wantInnerHits: false+- description: "blocked: aggregations with aggs key"+ body: '{"query": {"match_all": {}}, "aggs": {"status_count": {"terms": {"field": "status"}}}}'+ wantBlocked:+ - 0+ wantSourceDisabled: false+ wantInnerHits: false+- description: "blocked: aggregations with aggregations key"+ body: '{"aggregations": {"status_count": {"terms": {"field": "status"}}}}'+ wantBlocked:+ - 0+ wantSourceDisabled: false+ wantInnerHits: false+- description: "blocked: suggest"+ body: '{"suggest": {"my-suggest": {"text": "test", "term": {"field": "title"}}}}'+ wantBlocked:+ - 1+ wantSourceDisabled: false+ wantInnerHits: false+- description: "blocked: script_fields"+ body: '{"script_fields": {"my_field": {"script": "doc[''price''].value * 2"}}}'+ wantBlocked:+ - 2+ wantSourceDisabled: false+ wantInnerHits: false+- description: "blocked: runtime_mappings"+ body: '{"runtime_mappings": {"day_of_week": {"type": "keyword"}}}'+ wantBlocked:+ - 3+ wantSourceDisabled: false+ wantInnerHits: false+- description: "blocked: stored_fields"+ body: '{"stored_fields": ["title"]}'+ wantBlocked:+ - 4+ wantSourceDisabled: false+ wantInnerHits: false+- description: "blocked: docvalue_fields"+ body: '{"docvalue_fields": ["date"]}'+ wantBlocked:+ - 5+ wantSourceDisabled: false+ wantInnerHits: false+- description: "source disabled"+ body: '{"_source": false}'+ wantSourceDisabled: true+ wantInnerHits: false+- description: "source field list"+ body: '{"_source": ["name", "email"]}'+ wantSourceDisabled: false+ wantSourceFields:+ - "name"+ - "email"+ wantInnerHits: false+- description: "source includes/excludes"+ body: '{"_source": {"includes": ["name", "contact.*"]}}'+ wantSourceDisabled: false+ wantSourceFields:+ - "name"+ - "contact.*"+ wantInnerHits: false+- description: "fields parameter"+ body: '{"fields": ["created_at", "status"]}'+ wantSourceDisabled: false+ wantFields:+ - "created_at"+ - "status"+ wantInnerHits: false+- description: "highlight fields"+ body: '{"highlight": {"fields": {"message": {}, "title": {}}}}'+ wantSourceDisabled: false+ wantHighlight:+ - "message"+ - "title"+ wantInnerHits: false+- description: "sort fields"+ body: '{"sort": [{"date": "desc"}, "_score", {"price": {"order": "asc"}}]}'+ wantSourceDisabled: false+ wantSort:+ - "date"+ - "price"+ wantInnerHits: false+- description: "inner_hits present in nested query"+ body: '{"query": {"nested": {"path": "comments", "inner_hits": {}}}}'+ wantSourceDisabled: false+ wantInnerHits: true+- description: "inner_hits absent"+ body: '{"query": {"match": {"title": "test"}}}'+ wantSourceDisabled: false+ wantInnerHits: falsediff --git a/backend/plugin/parser/elasticsearch/test-data/masking_analyze_request.yaml b/backend/plugin/parser/elasticsearch/test-data/masking_analyze_request.yamlnew file mode 100644index 0000000000..0ea4d275df--- /dev/null+++ b/backend/plugin/parser/elasticsearch/test-data/masking_analyze_request.yaml@@ -0,0 +1,27 @@+# Test cases for AnalyzeRequest.+# MaskableAPI enum: APIUnsupported=0, APIMaskSearch=1, APIMaskGetDoc=2, APIMaskGetSource=3, APIMaskMGet=4, APIMaskExplain=5, APIBlocked=6+- description: "search with blocked feature"+ method: "POST"+ url: "/my-index/_search"+ body: '{"aggs": {"x": {"terms": {"field": "status"}}}}'+ wantAPI: 1+ wantIndex: "my-index"+ wantBlocked:+ - 0+- description: "get doc skips body analysis"+ method: "GET"+ url: "/my-index/_doc/1"+ body: '{"aggs": {"x": {}}}'+ wantAPI: 2+- description: "blocked endpoint"+ method: "POST"+ url: "/_sql"+ body: '{"query": "SELECT * FROM test"}'+ wantAPI: 6+- description: "search copies predicate fields"+ method: "POST"+ url: "/my-index/_search"+ body: '{"query": {"match": {"email": "[email protected]"}}}'+ wantAPI: 1+ wantPredicateFields:+ - "email"diff --git a/backend/plugin/parser/elasticsearch/test-data/masking_classify_api.yaml b/backend/plugin/parser/elasticsearch/test-data/masking_classify_api.yamlnew file mode 100644index 0000000000..8bc8b4b089--- /dev/null+++ b/backend/plugin/parser/elasticsearch/test-data/masking_classify_api.yaml@@ -0,0 +1,180 @@+# Test cases for classifyMaskableAPI.+# MaskableAPI enum: APIUnsupported=0, APIMaskSearch=1, APIMaskGetDoc=2, APIMaskGetSource=3, APIMaskMGet=4, APIMaskExplain=5, APIBlocked=6++# Supported: _search+- description: "POST _search"+ method: "POST"+ url: "/my-index/_search"+ wantAPI: 1+ wantIndex: "my-index"+- description: "GET _search"+ method: "GET"+ url: "/my-index/_search"+ wantAPI: 1+ wantIndex: "my-index"+- description: "POST _search with query params"+ method: "POST"+ url: "/my-index/_search?size=10"+ wantAPI: 1+ wantIndex: "my-index"++# Supported: _msearch+- description: "POST _msearch without index"+ method: "POST"+ url: "/_msearch"+ wantAPI: 1+ wantIndex: ""+- description: "POST _msearch with index"+ method: "POST"+ url: "/my-index/_msearch"+ wantAPI: 1+ wantIndex: "my-index"++# Supported: GET _doc+- description: "GET _doc"+ method: "GET"+ url: "/my-index/_doc/1"+ wantAPI: 2+ wantIndex: "my-index"++# Supported: GET _source+- description: "GET _source"+ method: "GET"+ url: "/my-index/_source/1"+ wantAPI: 3+ wantIndex: "my-index"++# Supported: _mget+- description: "POST _mget without index"+ method: "POST"+ url: "/_mget"+ wantAPI: 4+ wantIndex: ""+- description: "POST _mget with index"+ method: "POST"+ url: "/my-index/_mget"+ wantAPI: 4+ wantIndex: "my-index"+- description: "GET _mget without index"+ method: "GET"+ url: "/_mget"+ wantAPI: 4+ wantIndex: ""++# Supported: _explain+- description: "POST _explain"+ method: "POST"+ url: "/my-index/_explain/1"+ wantAPI: 5+ wantIndex: "my-index"++# Blocked endpoints+- description: "POST _async_search blocked"+ method: "POST"+ url: "/my-index/_async_search"+ wantAPI: 6+ wantIndex: "my-index"+- description: "GET _async_search status blocked"+ method: "GET"+ url: "/_async_search/abc123"+ wantAPI: 6+ wantIndex: ""+- description: "POST _search/scroll blocked"+ method: "POST"+ url: "/_search/scroll"+ wantAPI: 6+ wantIndex: ""+- description: "POST _search_template blocked"+ method: "POST"+ url: "/my-index/_search_template"+ wantAPI: 6+ wantIndex: "my-index"+- description: "POST _msearch/template blocked"+ method: "POST"+ url: "/_msearch/template"+ wantAPI: 6+ wantIndex: ""+- description: "POST _sql blocked"+ method: "POST"+ url: "/_sql"+ wantAPI: 6+ wantIndex: ""+- description: "POST _eql/search blocked"+ method: "POST"+ url: "/my-index/_eql/search"+ wantAPI: 6+ wantIndex: "my-index"+- description: "POST _esql/query blocked"+ method: "POST"+ url: "/_esql/query"+ wantAPI: 6+ wantIndex: ""+- description: "POST _terms_enum blocked"+ method: "POST"+ url: "/my-index/_terms_enum"+ wantAPI: 6+ wantIndex: "my-index"+- description: "GET _termvectors blocked"+ method: "GET"+ url: "/my-index/_termvectors/1"+ wantAPI: 6+ wantIndex: "my-index"++# OpenSearch blocked endpoints+- description: "POST _plugins/_asynchronous_search blocked (OpenSearch)"+ method: "POST"+ url: "/_plugins/_asynchronous_search"+ wantAPI: 6+ wantIndex: ""+- description: "POST _plugins/_sql blocked (OpenSearch)"+ method: "POST"+ url: "/_plugins/_sql"+ wantAPI: 6+ wantIndex: ""+- description: "POST _plugins/_ppl blocked (OpenSearch)"+ method: "POST"+ url: "/_plugins/_ppl"+ wantAPI: 6+ wantIndex: ""++# Unsupported (no user data)+- description: "HEAD _doc unsupported"+ method: "HEAD"+ url: "/my-index/_doc/1"+ wantAPI: 0+ wantIndex: ""+- description: "PUT _doc unsupported"+ method: "PUT"+ url: "/my-index/_doc/1"+ wantAPI: 0+ wantIndex: ""+- description: "DELETE _doc unsupported"+ method: "DELETE"+ url: "/my-index/_doc/1"+ wantAPI: 0+ wantIndex: ""+- description: "GET _cat/indices unsupported"+ method: "GET"+ url: "/_cat/indices"+ wantAPI: 0+ wantIndex: ""+- description: "GET _cluster/health unsupported"+ method: "GET"+ url: "/_cluster/health"+ wantAPI: 0+ wantIndex: ""+- description: "POST _count unsupported"+ method: "POST"+ url: "/my-index/_count"+ wantAPI: 0+ wantIndex: ""+- description: "POST _validate/query unsupported"+ method: "POST"+ url: "/my-index/_validate/query"+ wantAPI: 0+ wantIndex: ""+- description: "PUT index unsupported"+ method: "PUT"+ url: "/my-index"+ wantAPI: 0+ wantIndex: ""diff --git a/backend/plugin/parser/elasticsearch/test-data/masking_predicate_fields.yaml b/backend/plugin/parser/elasticsearch/test-data/masking_predicate_fields.yamlnew file mode 100644index 0000000000..ec728976e8--- /dev/null+++ b/backend/plugin/parser/elasticsearch/test-data/masking_predicate_fields.yaml@@ -0,0 +1,36 @@+# Test cases for extractPredicateFields (via analyzeRequestBody).+- description: "match query"+ body: '{"query": {"match": {"email": "[email protected]"}}}'+ wantFields:+ - "email"+- description: "term query"+ body: '{"query": {"term": {"status": "active"}}}'+ wantFields:+ - "status"+- description: "range query"+ body: '{"query": {"range": {"age": {"gte": 18}}}}'+ wantFields:+ - "age"+- description: "bool with must and filter"+ body: '{"query": {"bool": {"must": [{"match": {"name": "Alice"}}], "filter": [{"term": {"status": "active"}}]}}}'+ wantFields:+ - "name"+ - "status"+- description: "exists query"+ body: '{"query": {"exists": {"field": "email"}}}'+ wantFields:+ - "email"+- description: "nested query"+ body: '{"query": {"nested": {"path": "comments", "query": {"match": {"comments.text": "hello"}}}}}'+ wantFields:+ - "comments.text"+- description: "no query clause"+ body: '{"size": 10}'+ wantFields: []+- description: "match_all"+ body: '{"query": {"match_all": {}}}'+ wantFields: []+- description: "wildcard query"+ body: '{"query": {"wildcard": {"name": {"value": "ali*"}}}}'+ wantFields:+ - "name"diff --git a/frontend/src/utils/v1/instance.ts b/frontend/src/utils/v1/instance.tsindex 1f20a0f663..b163d0964b 100644--- a/frontend/src/utils/v1/instance.ts+++ b/frontend/src/utils/v1/instance.ts@@ -299,7 +299,9 @@ export const instanceV1MaskingForNoSQL = (instanceOrEngine: Instance | InstanceResource | Engine) => {const engine = engineOfInstanceV1(instanceOrEngine);- return [Engine.MONGODB, Engine.COSMOSDB].includes(engine);+ return [Engine.MONGODB, Engine.COSMOSDB, Engine.ELASTICSEARCH].includes(+ engine+ );};export const engineOfInstanceV1 = (