Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(cosmosdb): upgrade parser and add masking for all 13 query feature areas (#19670)
h3n4l71aa6f1ce386
diff --git a/backend/api/v1/catalog_masking_cosmosdb_test.go b/backend/api/v1/catalog_masking_cosmosdb_test.gonew file mode 100644index 0000000000..fb562e48de--- /dev/null+++ b/backend/api/v1/catalog_masking_cosmosdb_test.go@@ -0,0 +1,600 @@+package v1++import (+ "context"+ "encoding/json"+ "testing"++ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/component/masker"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ cosmosdbparser "github.com/bytebase/bytebase/backend/plugin/parser/cosmosdb"+)++// cosmosDBTestSchema returns a schema for a container with:+//+// name (string), email (string, masked), country (string), population (number),+// _ts (number), location (object, masked), contact.phone (string, masked), contact.city (string)+func cosmosDBTestSchema() *storepb.ObjectSchema {+ return &storepb.ObjectSchema{+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "name": {Type: storepb.ObjectSchema_STRING},+ "email": {Type: storepb.ObjectSchema_STRING, SemanticType: "bb.default"},+ "country": {Type: storepb.ObjectSchema_STRING},+ "population": {Type: storepb.ObjectSchema_NUMBER},+ "status": {Type: storepb.ObjectSchema_STRING},+ "age": {Type: storepb.ObjectSchema_NUMBER},+ "score": {Type: storepb.ObjectSchema_NUMBER},+ "_ts": {Type: storepb.ObjectSchema_NUMBER},+ "_etag": {Type: storepb.ObjectSchema_STRING},+ "location": {+ Type: storepb.ObjectSchema_OBJECT,+ SemanticType: "bb.default",+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "type": {Type: storepb.ObjectSchema_STRING},+ "coordinates": {Type: storepb.ObjectSchema_ARRAY},+ },+ },+ },+ },+ "contact": {+ Type: storepb.ObjectSchema_OBJECT,+ Kind: &storepb.ObjectSchema_StructKind_{+ StructKind: &storepb.ObjectSchema_StructKind{+ Properties: map[string]*storepb.ObjectSchema{+ "phone": {Type: storepb.ObjectSchema_STRING, SemanticType: "bb.default"},+ "city": {Type: storepb.ObjectSchema_STRING},+ },+ },+ },+ },+ },+ },+ },+ }+}++func cosmosDBMaskers() map[string]masker.Masker {+ return map[string]masker.Masker{+ "bb.default": masker.NewDefaultFullMasker(),+ }+}++// getCosmosDBQuerySpan parses a CosmosDB SQL query and returns the query span.+func getCosmosDBQuerySpan(t *testing.T, query string) *base.QuerySpan {+ t.Helper()+ span, err := cosmosdbparser.GetQuerySpan(context.TODO(), base.GetQuerySpanContext{}, base.Statement{Text: query}, "", "", false)+ require.NoError(t, err)+ return span+}++// maskCosmosDBResultsForTest replicates the maskResults logic without store/database dependencies.+// It handles both object and scalar (VALUE query) results.+func maskCosmosDBResultsForTest(span *base.QuerySpan, results []*v1pb.QueryResult, schema *storepb.ObjectSchema, maskers map[string]masker.Masker) error {+ if len(span.Results) != 1 {+ return errors.Errorf("expected 1 span result, got %d", len(span.Results))+ }+ for _, result := range results {+ for _, row := range result.Rows {+ if len(row.Values) != 1 {+ continue+ }+ value := row.Values[0].GetStringValue()+ if value == "" {+ continue+ }+ doc := make(map[string]any)+ if err := json.Unmarshal([]byte(value), &doc); err != nil {+ // VALUE query: the result is a scalar, not an object.+ // Check if any source field path is sensitive and mask the scalar directly.+ masked, maskErr := maskCosmosDBScalarValue(span, value, schema, maskers)+ if maskErr != nil {+ return maskErr+ }+ row.Values[0] = &v1pb.RowValue{+ Kind: &v1pb.RowValue_StringValue{StringValue: masked},+ }+ continue+ }+ maskedDoc, err := maskCosmosDB(span, doc, schema, maskers)+ if err != nil {+ return err+ }+ maskedValue, err := json.Marshal(maskedDoc)+ if err != nil {+ return err+ }+ row.Values[0] = &v1pb.RowValue{+ Kind: &v1pb.RowValue_StringValue{StringValue: string(maskedValue)},+ }+ }+ }+ return nil+}++// maskCosmosDBDoc parses the query, gets span, and masks the document.+func maskCosmosDBDoc(t *testing.T, query string, inputJSON string, schema *storepb.ObjectSchema, maskers map[string]masker.Masker) string {+ t.Helper()+ span := getCosmosDBQuerySpan(t, query)++ var doc map[string]any+ require.NoError(t, json.Unmarshal([]byte(inputJSON), &doc))++ masked, err := maskCosmosDB(span, doc, schema, maskers)+ require.NoError(t, err)++ out, err := json.Marshal(masked)+ require.NoError(t, err)+ return string(out)+}++// ---------------------------------------------------------------------------+// Test: Result masking with new syntax+// ---------------------------------------------------------------------------++func TestCosmosDBMaskingSelectTop(t *testing.T) {+ // SELECT TOP N should mask sensitive fields just like regular SELECT.+ query := `SELECT TOP 10 c.name, c.email FROM c`+ input := `{"name":"Alice","email":"[email protected]"}`+ want := `{"name":"Alice","email":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingSelectStar(t *testing.T) {+ // SELECT * masks all fields based on schema.+ query := `SELECT * FROM c`+ input := `{"name":"Alice","email":"[email protected]","country":"US"}`+ want := `{"name":"Alice","email":"******","country":"US"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingSelectTopStar(t *testing.T) {+ // SELECT TOP N * masks like SELECT *.+ query := `SELECT TOP 5 * FROM c`+ input := `{"name":"Alice","email":"[email protected]"}`+ want := `{"name":"Alice","email":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingFunctionsInSelect(t *testing.T) {+ // Functions in SELECT produce computed values. The function result column+ // (aliased) shouldn't match any schema field, so it should pass through unmasked.+ // Direct field references should still be masked.+ query := `SELECT c.name, c.email, UPPER(c.country) AS upperCountry FROM c`+ input := `{"name":"Alice","email":"[email protected]","upperCountry":"US"}`+ want := `{"name":"Alice","email":"******","upperCountry":"US"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingAggregation(t *testing.T) {+ // Aggregation result: COUNT(1) AS totalRecords.+ // The result is a computed value, not a direct field reference.+ query := `SELECT COUNT(1) AS totalRecords FROM c`+ input := `{"totalRecords":42}`+ want := `{"totalRecords":42}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingGroupBy(t *testing.T) {+ // GROUP BY with aggregation. Direct field (country) + function result (cnt).+ query := `SELECT c.country, COUNT(1) AS cnt FROM c GROUP BY c.country`+ input := `{"country":"US","cnt":5}`+ want := `{"country":"US","cnt":5}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingOrderBy(t *testing.T) {+ // ORDER BY doesn't affect masking.+ query := `SELECT c.name, c.email FROM c ORDER BY c.name ASC`+ input := `{"name":"Alice","email":"[email protected]"}`+ want := `{"name":"Alice","email":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingOffsetLimit(t *testing.T) {+ // OFFSET LIMIT doesn't affect masking.+ query := `SELECT c.name, c.email FROM c ORDER BY c.name OFFSET 0 LIMIT 10`+ input := `{"name":"Alice","email":"[email protected]"}`+ want := `{"name":"Alice","email":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingStringFunctions(t *testing.T) {+ // String functions in SELECT: UPPER, LOWER, LENGTH.+ query := `SELECT UPPER(c.name) AS upperName, LOWER(c.country) AS lowerCountry, LENGTH(c.name) AS nameLen FROM c`+ input := `{"upperName":"ALICE","lowerCountry":"us","nameLen":5}`+ want := `{"upperName":"ALICE","lowerCountry":"us","nameLen":5}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingMathFunctions(t *testing.T) {+ // Math functions in SELECT.+ query := `SELECT c.name, ROUND(c.score) AS roundedScore FROM c`+ input := `{"name":"Alice","roundedScore":95}`+ want := `{"name":"Alice","roundedScore":95}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingTypeCheckFunctions(t *testing.T) {+ // Type-check functions in SELECT.+ query := `SELECT c.name, IS_STRING(c.name) AS isStr, IS_NUMBER(c.population) AS isNum FROM c`+ input := `{"name":"Alice","isStr":true,"isNum":true}`+ want := `{"name":"Alice","isStr":true,"isNum":true}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingUnderscoreFields(t *testing.T) {+ // Underscore-prefixed fields like _ts, _etag should work with the new IDENTIFIER rule.+ query := `SELECT c._ts, c._etag, c.name FROM c`+ input := `{"_ts":1743702439,"_etag":"abc123","name":"Alice"}`+ want := `{"_ts":1743702439,"_etag":"abc123","name":"Alice"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingNestedFieldProjection(t *testing.T) {+ // Projecting nested sensitive field.+ query := `SELECT c.name, c.contact.phone AS phone FROM c`+ input := `{"name":"Alice","phone":"123-456"}`+ want := `{"name":"Alice","phone":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingDistinctValue(t *testing.T) {+ // SELECT DISTINCT VALUE returns scalar values, not objects.+ // The result is just a plain value like "US" instead of {"country":"US"}.+ // CosmosDB returns each value as a single-field JSON, but the masking+ // code needs to handle this correctly.+ query := `SELECT DISTINCT VALUE c.country FROM c`+ input := `{"country":"US"}`+ want := `{"country":"US"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingDistinctValueSensitive(t *testing.T) {+ // SELECT DISTINCT VALUE on a sensitive field.+ query := `SELECT DISTINCT VALUE c.email FROM c`+ input := `{"email":"[email protected]"}`+ want := `{"email":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingGeospatial(t *testing.T) {+ // Geospatial function with JSON object literal where source field IS sensitive.+ // ST_DISTANCE(c.location, ...) -- c.location is tagged with bb.default.+ // The distance result must be masked because it derives from sensitive location data.+ query := `SELECT c.name, ST_DISTANCE(c.location, {"type": "Point", "coordinates": [55.2708, 25.2048]}) AS dist FROM c`+ input := `{"name":"Alice","dist":1234.5}`+ want := `{"name":"Alice","dist":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingValueSensitive(t *testing.T) {+ // SELECT VALUE c.email returns scalar values from CosmosDB, not objects.+ // The driver returns each result as a raw JSON string (e.g., "\"[email protected]\"").+ // maskResults must handle non-object JSON values for VALUE queries.+ schema := cosmosDBTestSchema()+ maskers := cosmosDBMaskers()+ query := `SELECT VALUE c.email FROM container c`+ span := getCosmosDBQuerySpan(t, query)++ // Simulate the driver result: a scalar JSON string.+ row := &v1pb.QueryRow{+ Values: []*v1pb.RowValue{+ {Kind: &v1pb.RowValue_StringValue{StringValue: `"[email protected]"`}},+ },+ }+ result := &v1pb.QueryResult{+ ColumnNames: []string{"result"},+ ColumnTypeNames: []string{"TEXT"},+ Rows: []*v1pb.QueryRow{row},+ }++ err := maskCosmosDBResultsForTest(span, []*v1pb.QueryResult{result}, schema, maskers)+ require.NoError(t, err)+ require.Equal(t, `"******"`, row.Values[0].GetStringValue())+}++func TestCosmosDBMaskingValueNonSensitive(t *testing.T) {+ // SELECT VALUE c.name returns scalar values; c.name is not sensitive.+ schema := cosmosDBTestSchema()+ maskers := cosmosDBMaskers()+ query := `SELECT VALUE c.name FROM container c`+ span := getCosmosDBQuerySpan(t, query)++ row := &v1pb.QueryRow{+ Values: []*v1pb.RowValue{+ {Kind: &v1pb.RowValue_StringValue{StringValue: `"Alice"`}},+ },+ }+ result := &v1pb.QueryResult{+ ColumnNames: []string{"result"},+ ColumnTypeNames: []string{"TEXT"},+ Rows: []*v1pb.QueryRow{row},+ }++ err := maskCosmosDBResultsForTest(span, []*v1pb.QueryResult{result}, schema, maskers)+ require.NoError(t, err)+ require.Equal(t, `"Alice"`, row.Values[0].GetStringValue())+}++func TestCosmosDBMaskingValueCount(t *testing.T) {+ // SELECT VALUE COUNT(1) returns a number. No source field, not masked.+ schema := cosmosDBTestSchema()+ maskers := cosmosDBMaskers()+ query := `SELECT VALUE COUNT(1) FROM c`+ span := getCosmosDBQuerySpan(t, query)++ row := &v1pb.QueryRow{+ Values: []*v1pb.RowValue{+ {Kind: &v1pb.RowValue_StringValue{StringValue: `42`}},+ },+ }+ result := &v1pb.QueryResult{+ ColumnNames: []string{"result"},+ ColumnTypeNames: []string{"TEXT"},+ Rows: []*v1pb.QueryRow{row},+ }++ err := maskCosmosDBResultsForTest(span, []*v1pb.QueryResult{result}, schema, maskers)+ require.NoError(t, err)+ require.Equal(t, `42`, row.Values[0].GetStringValue())+}++// ---------------------------------------------------------------------------+// Test: Expression projections+// ---------------------------------------------------------------------------++func TestCosmosDBMaskingArithmeticProjection(t *testing.T) {+ // Arithmetic expression: c.population / 1000 AS popK.+ // Expression path is nil, alias "popK" not in schema, so not masked.+ query := `SELECT c.name, c.population / 1000 AS popK FROM c`+ input := `{"name":"Alice","popK":1}`+ want := `{"name":"Alice","popK":1}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingStringConcatProjection(t *testing.T) {+ // String concatenation of non-sensitive fields: c.name || " - " || c.country AS label.+ // Source fields (name, country) are not sensitive, so result is not masked.+ query := `SELECT c.name || " - " || c.country AS label FROM c`+ input := `{"label":"Alice - US"}`+ want := `{"label":"Alice - US"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingStringConcatWithSensitiveField(t *testing.T) {+ // String concatenation that includes a sensitive field.+ // c.name || " - " || c.email AS label -- c.email is sensitive.+ // Result must be masked because it derives from sensitive data.+ query := `SELECT c.name || " - " || c.email AS label FROM c`+ input := `{"label":"Alice - [email protected]"}`+ want := `{"label":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingNestedFunctionProjection(t *testing.T) {+ // Nested function calls: ROUND(StringToNumber(c.population)).+ // Expression path is nil, computed value not masked.+ query := `SELECT c.name, ROUND(StringToNumber(c.population)) AS rounded FROM c`+ input := `{"name":"Alice","rounded":1000}`+ want := `{"name":"Alice","rounded":1000}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingArithmeticOnFunctionProjection(t *testing.T) {+ // Arithmetic on function result: StringToNumber(c.population) / 1000.+ // Expression path is nil, alias not in schema, not masked.+ query := `SELECT CONCAT(c.name, " - ", c.country) AS label, StringToNumber(c.population) / 1000 AS populationInThousands FROM c`+ input := `{"label":"Alice - US","populationInThousands":1}`+ want := `{"label":"Alice - US","populationInThousands":1}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingExprWithSensitiveFieldAlias(t *testing.T) {+ // Expression alias matches a sensitive schema field name.+ // UPPER(c.name) AS email -- source field c.name is NOT sensitive.+ // Even though alias "email" is sensitive in the schema, the masking+ // should be driven by the source field (c.name), not the alias.+ // Since c.name is not sensitive, the result is NOT masked.+ query := `SELECT UPPER(c.name) AS email FROM c`+ input := `{"email":"ALICE"}`+ want := `{"email":"ALICE"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingExprMixedWithDirectFields(t *testing.T) {+ // Mix of direct field references and expressions.+ // Direct sensitive field (email) should be masked.+ // Expression results from non-sensitive fields (upperName, popK) should not be masked.+ query := `SELECT c.email, UPPER(c.name) AS upperName, c.population / 1000 AS popK FROM c`+ input := `{"email":"[email protected]","upperName":"ALICE","popK":1}`+ want := `{"email":"******","upperName":"ALICE","popK":1}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingExprOnSensitiveFieldNonMatchingAlias(t *testing.T) {+ // Function wrapping a sensitive field with a non-matching alias.+ // UPPER(c.email) AS upperEmail -- c.email IS sensitive.+ // The result must be masked because it derives from sensitive data,+ // regardless of the alias name.+ query := `SELECT UPPER(c.email) AS upperEmail FROM c`+ input := `{"upperEmail":"[email protected]"}`+ want := `{"upperEmail":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++func TestCosmosDBMaskingConcatWithSensitiveArg(t *testing.T) {+ // CONCAT with a sensitive field argument.+ // CONCAT(c.name, " <", c.email, ">") AS display -- c.email is sensitive.+ // Result must be masked because it contains sensitive data.+ query := `SELECT CONCAT(c.name, " <", c.email, ">") AS display FROM c`+ input := `{"display":"Alice <[email protected]>"}`+ want := `{"display":"******"}`++ got := maskCosmosDBDoc(t, query, input, cosmosDBTestSchema(), cosmosDBMaskers())+ requireJSONEqual(t, want, got)+}++// ---------------------------------------------------------------------------+// Test: Predicate blocking with new WHERE operators+// ---------------------------------------------------------------------------++func TestCosmosDBPredicateNotEqual(t *testing.T) {+ // WHERE with != on a sensitive field should be detected.+ query := `SELECT * FROM c WHERE c.email != "[email protected]"`+ span := getCosmosDBQuerySpan(t, query)+ schema := cosmosDBTestSchema()++ _, ok := span.PredicatePaths["c.email"]+ require.True(t, ok, "expected c.email in predicate paths")++ semanticType := getFirstSemanticTypeInPath(span.PredicatePaths["c.email"], schema)+ require.Equal(t, "bb.default", semanticType, "email should be sensitive in predicate")+}++func TestCosmosDBPredicateIn(t *testing.T) {+ // WHERE with IN on a sensitive field should be detected.+ query := `SELECT * FROM c WHERE c.email IN ("[email protected]", "[email protected]")`+ span := getCosmosDBQuerySpan(t, query)+ schema := cosmosDBTestSchema()++ _, ok := span.PredicatePaths["c.email"]+ require.True(t, ok, "expected c.email in predicate paths")++ semanticType := getFirstSemanticTypeInPath(span.PredicatePaths["c.email"], schema)+ require.Equal(t, "bb.default", semanticType, "email should be sensitive in predicate")+}++func TestCosmosDBPredicateBetween(t *testing.T) {+ // WHERE with BETWEEN on a non-sensitive field should NOT be blocked.+ query := `SELECT * FROM c WHERE c.age BETWEEN 18 AND 65`+ span := getCosmosDBQuerySpan(t, query)+ schema := cosmosDBTestSchema()++ _, ok := span.PredicatePaths["c.age"]+ require.True(t, ok, "expected c.age in predicate paths")++ semanticType := getFirstSemanticTypeInPath(span.PredicatePaths["c.age"], schema)+ require.Equal(t, "", semanticType, "age should not be sensitive")+}++func TestCosmosDBPredicateInNonSensitive(t *testing.T) {+ // WHERE with IN on a non-sensitive field should NOT be blocked.+ query := `SELECT * FROM c WHERE c.country IN ("US", "UK", "CA")`+ span := getCosmosDBQuerySpan(t, query)+ schema := cosmosDBTestSchema()++ _, ok := span.PredicatePaths["c.country"]+ require.True(t, ok, "expected c.country in predicate paths")++ semanticType := getFirstSemanticTypeInPath(span.PredicatePaths["c.country"], schema)+ require.Equal(t, "", semanticType, "country should not be sensitive")+}++func TestCosmosDBPredicateNotEqualNonSensitive(t *testing.T) {+ // WHERE with != on a non-sensitive field should NOT be blocked.+ query := `SELECT * FROM c WHERE c.status != "inactive"`+ span := getCosmosDBQuerySpan(t, query)+ schema := cosmosDBTestSchema()++ _, ok := span.PredicatePaths["c.status"]+ require.True(t, ok, "expected c.status in predicate paths")++ semanticType := getFirstSemanticTypeInPath(span.PredicatePaths["c.status"], schema)+ require.Equal(t, "", semanticType, "status should not be sensitive")+}++func TestCosmosDBPredicateNestedSensitive(t *testing.T) {+ // WHERE with nested sensitive field c.contact.phone.+ query := `SELECT * FROM c WHERE c.contact.phone = "123-456"`+ span := getCosmosDBQuerySpan(t, query)+ schema := cosmosDBTestSchema()++ _, ok := span.PredicatePaths["c.contact.phone"]+ require.True(t, ok, "expected c.contact.phone in predicate paths")++ semanticType := getFirstSemanticTypeInPath(span.PredicatePaths["c.contact.phone"], schema)+ require.Equal(t, "bb.default", semanticType, "contact.phone should be sensitive")+}++func TestCosmosDBPredicateFunctionInWhere(t *testing.T) {+ // Function used in WHERE clause should extract fields from arguments.+ query := `SELECT * FROM c WHERE CONTAINS(c.email, "example")`+ span := getCosmosDBQuerySpan(t, query)+ schema := cosmosDBTestSchema()++ _, ok := span.PredicatePaths["c.email"]+ require.True(t, ok, "expected c.email in predicate paths from function argument")++ semanticType := getFirstSemanticTypeInPath(span.PredicatePaths["c.email"], schema)+ require.Equal(t, "bb.default", semanticType, "email should be sensitive even in function")+}++func TestCosmosDBPredicateCompoundWhere(t *testing.T) {+ // Compound WHERE with multiple operators including new ones.+ query := `SELECT * FROM c WHERE c.country != "US" AND c._ts > 1743702439`+ span := getCosmosDBQuerySpan(t, query)++ _, ok := span.PredicatePaths["c.country"]+ require.True(t, ok, "expected c.country in predicate paths")++ _, ok = span.PredicatePaths["c._ts"]+ require.True(t, ok, "expected c._ts in predicate paths")+}diff --git a/backend/api/v1/catalog_masking_test.go b/backend/api/v1/catalog_masking_test.goindex d04d91df0f..852e088a62 100644--- a/backend/api/v1/catalog_masking_test.go+++ b/backend/api/v1/catalog_masking_test.go@@ -444,7 +444,7 @@ func TestWalkAndMaskJSON(t *testing.T) {err := json.Unmarshal([]byte(tc.input), &input)require.NoError(t, err, tc.description)- fieldPaths := make(map[string]*base.PathAST)+ fieldPaths := make(map[string][]*base.PathAST)if len(tc.fieldPathsElements) > 0 {for field, e := range tc.fieldPathsElements {ast := base.NewPathAST(base.NewItemSelector("container"))@@ -453,7 +453,7 @@ func TestWalkAndMaskJSON(t *testing.T) {next.SetNext(e[i])next = next.GetNext()}- fieldPaths[field] = ast+ fieldPaths[field] = []*base.PathAST{ast}}}got, err := walkAndMaskJSON(input, fieldPaths, tc.objectSchema, tc.semanticTypeToMasker)diff --git a/backend/api/v1/document_masking.go b/backend/api/v1/document_masking.goindex 1c47d37aa7..de799b02d9 100644--- a/backend/api/v1/document_masking.go+++ b/backend/api/v1/document_masking.go@@ -274,45 +274,75 @@ func maskDocumentString(document string, objectSchema *storepb.ObjectSchema, semreturn string(out), nil}-func walkAndMaskJSON(data map[string]any, fieldPaths map[string]*parserbase.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {+func walkAndMaskJSON(data map[string]any, fieldPaths map[string][]*parserbase.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (map[string]any, error) {result := make(map[string]any)for key, value := range data {- o := objectSchema- var ast *parserbase.PathAST- if fieldPaths != nil {- // Relocate the object schema cursor to the path position.- // Skip the first node because it always represents the container.- if path, ok := fieldPaths[key]; ok {- if path != nil && path.Root != nil {- astWoutContainer := parserbase.NewPathAST(path.Root.GetNext())- ast = astWoutContainer- }- }+ masked, err := maskJSONField(key, value, fieldPaths, objectSchema, semanticTypeToMasker)+ if err != nil {+ return nil, err}- if ast == nil || ast.Root == nil {- ast = parserbase.NewPathAST(parserbase.NewItemSelector(key))+ result[key] = masked+ }+ return result, nil+}++// maskJSONField masks a single field value based on its source field paths or direct schema lookup.+func maskJSONField(key string, value any, fieldPaths map[string][]*parserbase.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (any, error) {+ if paths, ok := fieldPaths[key]; ok && len(paths) > 0 {+ return maskFieldBySourcePaths(value, paths, objectSchema, semanticTypeToMasker)+ }+ return maskFieldByDirectLookup(key, value, objectSchema, semanticTypeToMasker)+}++// maskFieldBySourcePaths checks all source paths for sensitivity, then falls back to recursive walk.+func maskFieldBySourcePaths(value any, paths []*parserbase.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (any, error) {+ if maskedValue, masked, err := maskBySourcePaths(value, paths, objectSchema, semanticTypeToMasker); err != nil {+ return nil, err+ } else if masked {+ return maskedValue, nil+ }+ ast := stripContainerFromPath(paths[0])+ o, _ := getObjectSchemaByPath(objectSchema, ast)+ return walkAndMaskJSONRecursive(value, o, semanticTypeToMasker)+}++// maskFieldByDirectLookup looks up the key directly in the schema (used when no field paths are recorded).+func maskFieldByDirectLookup(key string, value any, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (any, error) {+ ast := parserbase.NewPathAST(parserbase.NewItemSelector(key))+ o, semanticType := getObjectSchemaByPath(objectSchema, ast)+ if semanticType != "" {+ if m, ok := semanticTypeToMasker[semanticType]; ok {+ return applyMaskerToJSONMember(value, m)}+ }+ return walkAndMaskJSONRecursive(value, o, semanticTypeToMasker)+}- var parentSemanticType string- o, parentSemanticType = getObjectSchemaByPath(o, ast)+// stripContainerFromPath returns a PathAST with the first node (container) removed.+func stripContainerFromPath(path *parserbase.PathAST) *parserbase.PathAST {+ if path != nil && path.Root != nil {+ return parserbase.NewPathAST(path.Root.GetNext())+ }+ return parserbase.NewPathAST(nil)+}++// maskBySourcePaths checks if any source field path resolves to a sensitive semantic type.+// Returns (maskedValue, true, nil) if masked, or (nil, false, nil) if no path is sensitive.+func maskBySourcePaths(value any, paths []*parserbase.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (any, bool, error) {+ for _, path := range paths {+ ast := stripContainerFromPath(path)+ _, parentSemanticType := getObjectSchemaByPath(objectSchema, ast)if parentSemanticType != "" {if m, ok := semanticTypeToMasker[parentSemanticType]; ok {maskedValue, err := applyMaskerToJSONMember(value, m)if err != nil {- return nil, err+ return nil, false, err}- result[key] = maskedValue- continue+ return maskedValue, true, nil}}-- fieldValue, err := walkAndMaskJSONRecursive(value, o, semanticTypeToMasker)- if err != nil {- return nil, err- }- result[key] = fieldValue}- return result, nil+ return nil, false, nil}func getObjectSchemaByPath(objectSchema *storepb.ObjectSchema, path *parserbase.PathAST) (*storepb.ObjectSchema, string) {@@ -588,7 +618,15 @@ func (*cosmosDBMasker) maskResults(}doc := make(map[string]any)if err := json.Unmarshal([]byte(value), &doc); err != nil {- return connect.NewError(connect.CodeInternal, errors.Errorf("failed to unmarshal document: %v", err))+ // VALUE query: result is a scalar, not a JSON object.+ masked, maskErr := maskCosmosDBScalarValue(spans[0], value, objectSchema, semanticTypeToMaskerMap)+ if maskErr != nil {+ return connect.NewError(connect.CodeInternal, errors.Errorf("failed to mask scalar value: %v", maskErr))+ }+ row.Values[0] = &v1pb.RowValue{+ Kind: &v1pb.RowValue_StringValue{StringValue: masked},+ }+ continue}maskedDoc, err := maskCosmosDB(spans[0], doc, objectSchema, semanticTypeToMaskerMap)if err != nil {@@ -615,6 +653,55 @@ func maskCosmosDB(span *parserbase.QuerySpan, data map[string]any, objectSchemareturn walkAndMaskJSON(data, span.Results[0].SourceFieldPaths, objectSchema, semanticTypeToMasker)}+// maskCosmosDBScalarValue handles VALUE query results where the JSON is a scalar+// (string, number, bool) rather than an object. It checks if any projected source+// field path resolves to a sensitive semantic type and masks the raw JSON accordingly.+func maskCosmosDBScalarValue(span *parserbase.QuerySpan, rawJSON string, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) (string, error) {+ if len(span.Results) != 1 {+ return rawJSON, nil+ }+ m := findMaskerForScalarValue(span.Results[0].SourceFieldPaths, objectSchema, semanticTypeToMasker)+ if m == nil {+ return rawJSON, nil+ }+ return applyMaskerToScalarJSON(rawJSON, m)+}++// findMaskerForScalarValue checks all source field paths across all projected fields+// and returns the first masker found for a sensitive semantic type, or nil.+func findMaskerForScalarValue(fieldPaths map[string][]*parserbase.PathAST, objectSchema *storepb.ObjectSchema, semanticTypeToMasker map[string]masker.Masker) masker.Masker {+ for _, paths := range fieldPaths {+ for _, path := range paths {+ ast := stripContainerFromPath(path)+ _, semanticType := getObjectSchemaByPath(objectSchema, ast)+ if semanticType == "" {+ continue+ }+ if m, ok := semanticTypeToMasker[semanticType]; ok {+ return m+ }+ }+ }+ return nil+}++// applyMaskerToScalarJSON parses a raw JSON scalar, masks it, and re-serializes.+func applyMaskerToScalarJSON(rawJSON string, m masker.Masker) (string, error) {+ var parsed any+ if err := json.Unmarshal([]byte(rawJSON), &parsed); err != nil {+ return rawJSON, nil //nolint:nilerr // Unparseable JSON: pass through as-is.+ }+ masked, err := applyMaskerToJSONMember(parsed, m)+ if err != nil {+ return "", err+ }+ out, err := json.Marshal(masked)+ if err != nil {+ return "", err+ }+ return string(out), nil+}+// ---------------------------------------------------------------------------// MongoDB// ---------------------------------------------------------------------------diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.goindex 2b03cb4ed3..f5cb10790c 100644--- a/backend/plugin/parser/base/span.go+++ b/backend/plugin/parser/base/span.go@@ -83,8 +83,9 @@ type QuerySpanResult struct {SourceColumns SourceColumnSet// IsPlainField indicates whether the field is a plain column reference (true) or an expression (false).IsPlainField bool- // SourceFieldPaths is used for Cosmos DB only, to store the path of the field, the root is the container name.- SourceFieldPaths map[string]*PathAST+ // SourceFieldPaths is used for Cosmos DB only, to store the source field paths for each projected output name.+ // Multiple paths per key support expression projections (e.g., CONCAT(c.name, c.email) references two fields).+ SourceFieldPaths map[string][]*PathAST// SelectAsterisk indicates whether the field is selected by asterisk, used by Cosmos DB.SelectAsterisk bool}@@ -375,12 +376,14 @@ func (s *QuerySpan) ToYaml() *YamlQuerySpan {}return 0})- for k, v := range result.SourceFieldPaths {- s, _ := v.String()- yamlResult.SourceFieldPaths = append(yamlResult.SourceFieldPaths, YamlQuerySpanResultSourceFieldPaths{- Name: k,- Path: s,- })+ for k, paths := range result.SourceFieldPaths {+ for _, v := range paths {+ s, _ := v.String()+ yamlResult.SourceFieldPaths = append(yamlResult.SourceFieldPaths, YamlQuerySpanResultSourceFieldPaths{+ Name: k,+ Path: s,+ })+ }}slices.SortFunc(yamlResult.SourceFieldPaths, func(i, j YamlQuerySpanResultSourceFieldPaths) int {if i.Name < j.Name {diff --git a/backend/plugin/parser/cosmosdb/query_span.go b/backend/plugin/parser/cosmosdb/query_span.goindex b392370cb9..4d54616d4e 100644--- a/backend/plugin/parser/cosmosdb/query_span.go+++ b/backend/plugin/parser/cosmosdb/query_span.go@@ -2,8 +2,6 @@ package cosmosdbimport ("context"- "log/slog"- "strconv""github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/cosmosdb"@@ -65,126 +63,127 @@ type querySpanResultListener struct {}func (l *querySpanResultListener) EnterSelect(ctx *parser.SelectContext) {- // TODO(zp): Considering the case of multiple from sources once we support it.if ctx.Select_clause().Select_specification().MULTIPLY_OPERATOR() != nil || ctx.From_clause() == nil {l.result = []base.QuerySpanResult{{Name: "",- SourceFieldPaths: make(map[string]*base.PathAST),+ SourceFieldPaths: make(map[string][]*base.PathAST),SelectAsterisk: true,},}return}- var originalContainerName string- var fromIdentifier string- fromClause := ctx.From_clause()- if i := fromClause.From_specification().From_source().Container_expression().Container_name().IDENTIFIER(); i != nil {- originalContainerName = i.GetText()- }- // Alias in the from source will shadow the original identifier.- if i := fromClause.From_specification().From_source().Container_expression().IDENTIFIER(); i != nil {- fromIdentifier = i.GetText()- }-- sourceFieldPath := make(map[string]*base.PathAST)- objectProperties := ctx.Select_clause().Select_specification().Object_property_list().AllObject_property()- for _, property := range objectProperties {- path, name := extractPathFromObjectProperty(property, originalContainerName, fromIdentifier)- if len(path) == 0 {+ containerName, fromAlias := extractFromNames(ctx.From_clause())+ sourceFieldPaths := make(map[string][]*base.PathAST)+ for _, property := range ctx.Select_clause().Select_specification().Object_property_list().AllObject_property() {+ paths, name := extractPathsFromObjectProperty(property, containerName, fromAlias)+ if name == "" {continue}- ast := base.NewPathAST(path[0])- next := ast.Root- for i := 1; i < len(path); i++ {- next.SetNext(path[i])- next = next.GetNext()+ for _, path := range paths {+ if len(path) == 0 {+ continue+ }+ sourceFieldPaths[name] = append(sourceFieldPaths[name], buildPathAST(path))}- sourceFieldPath[name] = ast}l.result = []base.QuerySpanResult{{Name: "",- SourceFieldPaths: sourceFieldPath,+ SourceFieldPaths: sourceFieldPaths,SelectAsterisk: false,},}}-func extractPathFromObjectProperty(ctx parser.IObject_propertyContext, originalContainerName string, fromAlias string) ([]base.SelectorNode, string) {+// extractPathsFromObjectProperty extracts all source field paths and the output name+// from a SELECT object_property.+func extractPathsFromObjectProperty(ctx parser.IObject_propertyContext, containerName, fromAlias string) ([][]base.SelectorNode, string) {if ctx == nil {return nil, ""}-- path := extractPathFromScalarExpression(ctx.Scalar_expression(), originalContainerName, fromAlias)+ paths := extractAllFieldPaths(ctx.Scalar_expression(), containerName, fromAlias)var propertyName stringif ctx.Property_alias() != nil {- propertyName = ctx.Property_alias().IDENTIFIER().GetText()+ propertyName = ctx.Property_alias().Identifier().GetText()+ }+ if propertyName == "" && len(paths) == 1 && len(paths[0]) > 0 {+ propertyName = paths[0][len(paths[0])-1].GetIdentifier()}+ return paths, propertyName+}- if propertyName == "" {- // If the property alias is not specified, we will use the last path element as the property name.- if len(path) > 0 {- last := path[len(path)-1]- propertyName = last.GetIdentifier()- }+// extractAllFieldPaths recursively collects ALL field paths referenced in a scalar expression.+func extractAllFieldPaths(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ if ctx == nil {+ return nil}- return path, propertyName+ // Input alias (leaf).+ if ctx.Input_alias() != nil {+ return resolveInputAlias(ctx, containerName, fromAlias)+ }+ // Property access: a.b+ if ctx.DOT_SYMBOL() != nil {+ return extractFieldPathsDot(ctx, containerName, fromAlias)+ }+ // Bracket access: a["b"] or a[0]+ if ctx.LS_BRACKET_SYMBOL() != nil && ctx.IN_SYMBOL() == nil {+ return extractFieldPathsBracket(ctx, containerName, fromAlias)+ }+ // Unary / NOT+ if ctx.Unary_operator() != nil || (ctx.NOT_SYMBOL() != nil && len(ctx.AllScalar_expression()) == 1) {+ return extractAllFieldPaths(ctx.Scalar_expression(0), containerName, fromAlias)+ }+ // Function calls+ if ctx.Scalar_function_expression() != nil {+ return extractFieldPathsFromFunction(ctx.Scalar_function_expression(), containerName, fromAlias)+ }+ // Binary/multi-child expressions: collect from all children.+ if allExprs := ctx.AllScalar_expression(); len(allExprs) >= 2 {+ return collectFieldPathsFromAll(allExprs, containerName, fromAlias)+ }+ return nil}-func extractPathFromScalarExpression(ctx parser.IScalar_expressionContext, originalContainerName string, fromAlias string) []base.SelectorNode {+func extractFieldPathsDot(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ paths := extractAllFieldPaths(ctx.Scalar_expression(0), containerName, fromAlias)+ propName := ctx.Property_name().Identifier().GetText()+ for i := range paths {+ paths[i] = append(paths[i], base.NewItemSelector(propName))+ }+ return paths+}++func extractFieldPathsBracket(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ paths := extractAllFieldPaths(ctx.Scalar_expression(0), containerName, fromAlias)+ for i := range paths {+ appendBracketSelector(ctx, paths, i)+ }+ return paths+}++func extractFieldPathsFromFunction(ctx parser.IScalar_function_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {if ctx == nil {return nil}-+ var exprs []parser.IScalar_expressionContextswitch {- case ctx.Input_alias() != nil:- name := ctx.Input_alias().IDENTIFIER().GetText()- if fromAlias != "" && name == fromAlias {- name = originalContainerName- }- return []base.SelectorNode{- base.NewItemSelector(name),- }- case ctx.DOT_SYMBOL() != nil:- // Most usual case like a.b.c.d.- path := extractPathFromScalarExpression(ctx.Scalar_expression(), originalContainerName, fromAlias)- path = append(path, base.NewItemSelector(ctx.Property_name().IDENTIFIER().GetText()))-- return path- case ctx.LS_BRACKET_SYMBOL() != nil:- path := extractPathFromScalarExpression(ctx.Scalar_expression(), originalContainerName, fromAlias)- switch {- case ctx.DOUBLE_QUOTE_STRING_LITERAL() != nil:- text := ctx.DOUBLE_QUOTE_STRING_LITERAL().GetText()- if len(text) > 1 {- text = text[1 : len(text)-1]- }- path = append(path, base.NewItemSelector(text))- case ctx.Array_index() != nil:- if len(path) == 0 {- break- }- index, err := strconv.Atoi(ctx.Array_index().GetText())- if err != nil {- slog.Warn("cannot convert array index to int", slog.String("index", ctx.Array_index().GetText()))- break- }- // Rebuild the ast because of the different level of array index and array name.- last := path[len(path)-1]- path[len(path)-1] = base.NewArraySelector(last.GetIdentifier(), index)- default:- // Unsupported bracket expression type- }-- return path- case ctx.Unary_operator() != nil:- return extractPathFromScalarExpression(ctx.Scalar_expression(), originalContainerName, fromAlias)+ case ctx.Udf_scalar_function_expression() != nil:+ exprs = ctx.Udf_scalar_function_expression().AllScalar_expression()+ case ctx.Builtin_function_expression() != nil:+ exprs = ctx.Builtin_function_expression().AllScalar_expression()default:- // Unsupported scalar expression type+ return nil}+ return collectFieldPathsFromAll(exprs, containerName, fromAlias)+}- return nil+func collectFieldPathsFromAll(exprs []parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ var all [][]base.SelectorNode+ for _, expr := range exprs {+ all = append(all, extractAllFieldPaths(expr, containerName, fromAlias)...)+ }+ return all}diff --git a/backend/plugin/parser/cosmosdb/query_span_helpers.go b/backend/plugin/parser/cosmosdb/query_span_helpers.gonew file mode 100644index 0000000000..73934fe04c--- /dev/null+++ b/backend/plugin/parser/cosmosdb/query_span_helpers.go@@ -0,0 +1,73 @@+package cosmosdb++import (+ "log/slog"+ "strconv"++ parser "github.com/bytebase/parser/cosmosdb"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// extractFromNames returns the container name and from-clause alias.+func extractFromNames(fromClause parser.IFrom_clauseContext) (containerName, fromAlias string) {+ containerExpr := fromClause.From_specification().From_source().Container_expression()+ if i := containerExpr.Container_name().Identifier(); i != nil {+ containerName = i.GetText()+ }+ if i := containerExpr.Identifier(); i != nil {+ fromAlias = i.GetText()+ }+ return containerName, fromAlias+}++// resolveInputAlias returns a single-element path for an input alias, resolving from-clause alias.+func resolveInputAlias(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ name := ctx.Input_alias().Identifier().GetText()+ if fromAlias != "" && name == fromAlias {+ name = containerName+ }+ return [][]base.SelectorNode{{base.NewItemSelector(name)}}+}++// buildPathAST converts a slice of SelectorNodes into a linked PathAST.+func buildPathAST(path []base.SelectorNode) *base.PathAST {+ ast := base.NewPathAST(path[0])+ next := ast.Root+ for i := 1; i < len(path); i++ {+ next.SetNext(path[i])+ next = next.GetNext()+ }+ return ast+}++// unquote strips the first and last character from a quoted string literal.+func unquote(s string) string {+ if len(s) > 1 {+ return s[1 : len(s)-1]+ }+ return s+}++// appendBracketSelector appends the bracket-access selector (string key or array index)+// to the path at index i. Shared by query_span.go and query_span_predicate.go.+func appendBracketSelector(ctx parser.IScalar_expressionContext, paths [][]base.SelectorNode, i int) {+ switch {+ case ctx.DOUBLE_QUOTE_STRING_LITERAL() != nil:+ paths[i] = append(paths[i], base.NewItemSelector(unquote(ctx.DOUBLE_QUOTE_STRING_LITERAL().GetText())))+ case ctx.SINGLE_QUOTE_STRING_LITERAL() != nil:+ paths[i] = append(paths[i], base.NewItemSelector(unquote(ctx.SINGLE_QUOTE_STRING_LITERAL().GetText())))+ case ctx.Array_index() != nil:+ if len(paths[i]) == 0 {+ return+ }+ index, err := strconv.Atoi(ctx.Array_index().GetText())+ if err != nil {+ slog.Warn("cannot convert array index to int", slog.String("index", ctx.Array_index().GetText()))+ return+ }+ last := paths[i][len(paths[i])-1]+ paths[i][len(paths[i])-1] = base.NewArraySelector(last.GetIdentifier(), index)+ default:+ }+}diff --git a/backend/plugin/parser/cosmosdb/query_span_predicate.go b/backend/plugin/parser/cosmosdb/query_span_predicate.goindex 397bf492cf..f969e36063 100644--- a/backend/plugin/parser/cosmosdb/query_span_predicate.go+++ b/backend/plugin/parser/cosmosdb/query_span_predicate.go@@ -2,7 +2,6 @@ package cosmosdbimport ("log/slog"- "strconv"parser "github.com/bytebase/parser/cosmosdb"@@ -18,7 +17,6 @@ type querySpanPredicatePathsListener struct {}func (l *querySpanPredicatePathsListener) EnterSelect(ctx *parser.SelectContext) {- // Extracting predicate fields from where clause.whereClause := ctx.Where_clause()if whereClause == nil {return@@ -28,141 +26,122 @@ func (l *querySpanPredicatePathsListener) EnterSelect(ctx *parser.SelectContext)return}- var originalContainerName string- var fromIdentifier string- if i := fromClause.From_specification().From_source().Container_expression().Container_name().IDENTIFIER(); i != nil {- originalContainerName = i.GetText()- }- // Alias in the from source will shadow the original identifier.- if i := fromClause.From_specification().From_source().Container_expression().IDENTIFIER(); i != nil {- fromIdentifier = i.GetText()- }-- predicateFields := extractPredicateFieldsFromWhereClause(whereClause, originalContainerName, fromIdentifier)- l.predicatePaths = predicateFields+ originalContainerName, fromIdentifier := extractFromNames(fromClause)+ l.predicatePaths = buildPredicatePaths(whereClause.Scalar_expression(), originalContainerName, fromIdentifier)}-func extractPredicateFieldsFromWhereClause(ctx parser.IWhere_clauseContext, originalContainerName string, fromAlias string) map[string]*base.PathAST {- scalarExpression := ctx.Scalar_expression_in_where()-- paths := extractPredicateFieldsFromScalarExpressionInWhere(scalarExpression, originalContainerName, fromAlias)-+func buildPredicatePaths(expr parser.IScalar_expressionContext, containerName, fromAlias string) map[string]*base.PathAST {+ paths := extractPredicateFields(expr, containerName, fromAlias)r := make(map[string]*base.PathAST)for _, path := range paths {if len(path) == 0 {continue}-- ast := base.NewPathAST(path[0])- current := ast.Root- for i := 1; i < len(path); i++ {- current.SetNext(path[i])- current = current.GetNext()- }-+ ast := buildPathAST(path)str, err := ast.String()if err != nil {slog.Warn("failed to convert path ast to string", log.BBError(err))}r[str] = ast}-return r}-func extractPredicateFieldsFromScalarExpressionInWhere(ctx parser.IScalar_expression_in_whereContext, originalContainerName string, fromAlias string) [][]base.SelectorNode {+// extractPredicateFields collects all field paths referenced in a scalar expression used as a predicate.+func extractPredicateFields(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {if ctx == nil {return nil}+ // Input alias (leaf field reference).+ if ctx.Input_alias() != nil {+ return resolveInputAlias(ctx, containerName, fromAlias)+ }+ // Property access: a.b.c+ if ctx.DOT_SYMBOL() != nil {+ return extractPredicateDotAccess(ctx, containerName, fromAlias)+ }+ // Bracket access: a["b"] or a[0] (but not IN expression which also uses brackets).+ if ctx.LS_BRACKET_SYMBOL() != nil && ctx.IN_SYMBOL() == nil {+ return extractPredicateBracketAccess(ctx, containerName, fromAlias)+ }+ // Unary operator or standalone NOT.+ if ctx.Unary_operator() != nil || (ctx.NOT_SYMBOL() != nil && len(ctx.AllScalar_expression()) == 1) {+ return extractPredicateFields(ctx.Scalar_expression(0), containerName, fromAlias)+ }+ // Parenthesized expression (not subquery, not EXISTS).+ if ctx.LR_BRACKET_SYMBOL() != nil && ctx.Select_() == nil && ctx.EXISTS_SYMBOL() == nil {+ return extractPredicateFields(ctx.Scalar_expression(0), containerName, fromAlias)+ }+ // Function calls.+ if ctx.Scalar_function_expression() != nil {+ return extractPredicateFieldsFromFunction(ctx.Scalar_function_expression(), containerName, fromAlias)+ }+ // Ternary operator: cond ? a : b+ if ctx.QUESTION_MARK_SYMBOL() != nil {+ return collectPredicateFieldsFromAll(ctx.AllScalar_expression(), containerName, fromAlias)+ }+ // Binary operators (comparison, arithmetic, bitwise, logical, IN, BETWEEN, LIKE, concat).+ // All share the same logic: collect fields from all child expressions.+ if isBinaryOrMultiChildExpression(ctx) {+ return collectPredicateFieldsFromAll(ctx.AllScalar_expression(), containerName, fromAlias)+ }+ return nil+}++func extractPredicateDotAccess(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ paths := extractPredicateFields(ctx.Scalar_expression(0), containerName, fromAlias)+ propName := ctx.Property_name().Identifier().GetText()+ for i := range paths {+ paths[i] = append(paths[i], base.NewItemSelector(propName))+ }+ return paths+}++func extractPredicateBracketAccess(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ paths := extractPredicateFields(ctx.Scalar_expression(0), containerName, fromAlias)+ for i := range paths {+ appendBracketSelector(ctx, paths, i)+ }+ return paths+}++func extractPredicateFieldsFromFunction(ctx parser.IScalar_function_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ var exprs []parser.IScalar_expressionContextswitch {- case ctx.Input_alias() != nil:- name := ctx.Input_alias().IDENTIFIER().GetText()- if fromAlias != "" && name == fromAlias {- name = originalContainerName- }- return [][]base.SelectorNode{- {- base.NewItemSelector(name),- },- }- case ctx.AND_SYMBOL() != nil, ctx.OR_SYMBOL() != nil:- allScalarExpressionInWheres := ctx.AllScalar_expression_in_where()- var allPaths [][]base.SelectorNode- for _, expr := range allScalarExpressionInWheres {- paths := extractPredicateFieldsFromScalarExpressionInWhere(expr, originalContainerName, fromAlias)- allPaths = append(allPaths, paths...)- }- return allPaths- case ctx.DOT_SYMBOL() != nil:- // Most usual case like a.b.c.d.- paths := extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(0), originalContainerName, fromAlias)- for i := range paths {- paths[i] = append(paths[i], base.NewItemSelector(ctx.Property_name().IDENTIFIER().GetText()))- }- return paths- case ctx.LS_BRACKET_SYMBOL() != nil:- paths := extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(0), originalContainerName, fromAlias)- for i := range paths {- switch {- case ctx.Property_name() != nil:- paths[i] = append(paths[i], base.NewItemSelector(ctx.Property_name().IDENTIFIER().GetText()))- paths[i] = append(paths[i], base.NewItemSelector(ctx.Array_index().GetText()))- case ctx.Array_index() != nil:- if len(paths[i]) == 0 {- break- }- index, err := strconv.Atoi(ctx.Array_index().GetText())- if err != nil {- slog.Warn("cannot convert array index to int", slog.String("index", ctx.Array_index().GetText()))- break- }- // Rebuild the ast because of the different level of array index and array name.- last := paths[i][len(paths[i])-1]- paths[i][len(paths[i])-1] = base.NewArraySelector(last.GetIdentifier(), index)- default:- // Do nothing for other cases- }- }- return paths- case ctx.Unary_operator() != nil:- return extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(0), originalContainerName, fromAlias)- case ctx.Binary_operator() != nil:- left := extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(0), originalContainerName, fromAlias)- right := extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(1), originalContainerName, fromAlias)- return append(left, right...)- case ctx.QUESTION_MARK_SYMBOL() != nil:- left := extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(0), originalContainerName, fromAlias)- mid := extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(1), originalContainerName, fromAlias)- right := extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(2), originalContainerName, fromAlias)- return append(append(left, mid...), right...)- case ctx.Scalar_function_expression() != nil:- switch {- case ctx.Scalar_function_expression().Udf_scalar_function_expression() != nil:- allScalarExpressionInWheres := ctx.Scalar_function_expression().Udf_scalar_function_expression().AllScalar_expression_in_where()- var paths [][]base.SelectorNode- for _, expr := range allScalarExpressionInWheres {- path := extractPredicateFieldsFromScalarExpressionInWhere(expr, originalContainerName, fromAlias)- paths = append(paths, path...)- }- return paths- case ctx.Scalar_function_expression().Builtin_function_expression() != nil:- allScalarExpressionInWheres := ctx.Scalar_function_expression().Builtin_function_expression().AllScalar_expression_in_where()- var paths [][]base.SelectorNode- for _, expr := range allScalarExpressionInWheres {- path := extractPredicateFieldsFromScalarExpressionInWhere(expr, originalContainerName, fromAlias)- paths = append(paths, path...)- }- return paths- default:- // Other scalar function expression types- return nil- }- case ctx.LR_BRACKET_SYMBOL() != nil:- return extractPredicateFieldsFromScalarExpressionInWhere(ctx.Scalar_expression_in_where(0), originalContainerName, fromAlias)+ case ctx.Udf_scalar_function_expression() != nil:+ exprs = ctx.Udf_scalar_function_expression().AllScalar_expression()+ case ctx.Builtin_function_expression() != nil:+ exprs = ctx.Builtin_function_expression().AllScalar_expression()default:- // Return nil for unhandled cases+ return nil}+ return collectPredicateFieldsFromAll(exprs, containerName, fromAlias)+}- return nil+// isBinaryOrMultiChildExpression returns true for binary operators and multi-child expressions+// that all share the same extraction logic: collect fields from all child scalar expressions.+func isBinaryOrMultiChildExpression(ctx parser.IScalar_expressionContext) bool {+ return ctx.Comparison_operator() != nil ||+ ctx.Multiplicative_operator() != nil ||+ ctx.Additive_operator() != nil ||+ ctx.Shift_operator() != nil ||+ ctx.BIT_AND_SYMBOL() != nil ||+ ctx.BIT_XOR_SYMBOL() != nil ||+ ctx.BIT_OR_SYMBOL() != nil ||+ ctx.DOUBLE_BAR_SYMBOL() != nil ||+ ctx.LIKE_SYMBOL() != nil ||+ ctx.IN_SYMBOL() != nil ||+ ctx.BETWEEN_SYMBOL() != nil ||+ // AND/OR logical operators. BETWEEN also sets AND_SYMBOL, but BETWEEN is checked above.+ ctx.AND_SYMBOL() != nil ||+ ctx.OR_SYMBOL() != nil+}++func collectPredicateFieldsFromAll(exprs []parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {+ var all [][]base.SelectorNode+ for _, expr := range exprs {+ all = append(all, extractPredicateFields(expr, containerName, fromAlias)...)+ }+ return all}diff --git a/backend/plugin/parser/cosmosdb/query_span_predicate_test.go b/backend/plugin/parser/cosmosdb/query_span_predicate_test.goindex 9501f07516..06ecf192d1 100644--- a/backend/plugin/parser/cosmosdb/query_span_predicate_test.go+++ b/backend/plugin/parser/cosmosdb/query_span_predicate_test.go@@ -32,6 +32,63 @@ func TestPredicateInWhere(t *testing.T) {"container.age": true,},},+ {+ statement: `SELECT * FROM container c WHERE c.status != "inactive"`,+ predicatePaths: map[string]bool{+ "container.status": true,+ },+ },+ {+ statement: `SELECT * FROM container c WHERE c.country IN ("US", "UK", "CA")`,+ predicatePaths: map[string]bool{+ "container.country": true,+ },+ },+ {+ statement: `SELECT * FROM container c WHERE c.population BETWEEN 100000 AND 5000000`,+ predicatePaths: map[string]bool{+ "container.population": true,+ },+ },+ {+ statement: `SELECT * FROM container c WHERE c.country != "US" AND c._ts > 1000`,+ predicatePaths: map[string]bool{+ "container.country": true,+ "container._ts": true,+ },+ },+ {+ statement: `SELECT * FROM container c WHERE CONTAINS(c.name, "test")`,+ predicatePaths: map[string]bool{+ "container.name": true,+ },+ },+ {+ statement: `SELECT * FROM container c WHERE c.name LIKE "Alice%"`,+ predicatePaths: map[string]bool{+ "container.name": true,+ },+ },+ {+ statement: `SELECT * FROM container c WHERE NOT c.active`,+ predicatePaths: map[string]bool{+ "container.active": true,+ },+ },+ {+ statement: `SELECT * FROM container c WHERE c.email = "x" OR c.name = "y"`,+ predicatePaths: map[string]bool{+ "container.email": true,+ "container.name": true,+ },+ },+ {+ statement: `SELECT * FROM container c WHERE c.firstName || " " || c.lastName = "Alice Smith"`,+ predicatePaths: map[string]bool{+ "container.firstName": true,+ "container.lastName": true,+ },+ },}a := require.New(t)diff --git a/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml b/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yamlindex 27e6cce907..8d9338dd37 100644--- a/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml+++ b/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml@@ -1,4 +1,4 @@-- description: Test for declare temp table+- description: basic field projection with aliasstatement: select c.category, c.salary["value"] FROM container cquerySpan:type: 0@@ -15,3 +15,234 @@sourcecolumns: []predicatecolumns: []predicatepaths: []+- description: SELECT TOP with star+ statement: SELECT TOP 10 * FROM c+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: true+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT TOP with field projection+ statement: SELECT TOP 5 c.name, c.email FROM container c+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: email+ path: container.email+ - name: name+ path: container.name+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT with ORDER BY+ statement: SELECT c.name, c.population FROM container c ORDER BY c.population DESC+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: name+ path: container.name+ - name: population+ path: container.population+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT with OFFSET LIMIT+ statement: SELECT c.name FROM container c ORDER BY c.name OFFSET 0 LIMIT 10+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: name+ path: container.name+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT with function in projection+ statement: SELECT UPPER(c.name) AS upperName, c.email FROM container c+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: email+ path: container.email+ - name: upperName+ path: container.name+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT with aggregation+ statement: SELECT COUNT(1) AS totalRecords FROM c+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT with GROUP BY+ statement: SELECT c.country, COUNT(1) AS cnt FROM container c GROUP BY c.country+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: country+ path: container.country+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT DISTINCT VALUE+ statement: SELECT DISTINCT VALUE c.country FROM container c+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: country+ path: container.country+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT VALUE+ statement: SELECT VALUE c.name FROM container c+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: name+ path: container.name+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: SELECT with underscore fields+ statement: SELECT c._ts, c._etag, c.name FROM container c+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: _etag+ path: container._etag+ - name: _ts+ path: container._ts+ - name: name+ path: container.name+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths: []+- description: WHERE with not-equal operator+ statement: SELECT * FROM container c WHERE c.status != "inactive"+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: true+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths:+ - container.status+- description: WHERE with IN operator+ statement: SELECT * FROM container c WHERE c.country IN ("US", "UK", "CA")+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: true+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths:+ - container.country+- description: WHERE with BETWEEN operator+ statement: SELECT * FROM container c WHERE c.population BETWEEN 100000 AND 5000000+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: true+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths:+ - container.population+- description: WHERE with compound condition using new operators+ statement: SELECT c.name, c.email FROM container c WHERE c.country != "US" AND c._ts > 1000+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths:+ - name: email+ path: container.email+ - name: name+ path: container.name+ selectasterisk: false+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths:+ - container._ts+ - container.country+- description: WHERE with function call+ statement: SELECT * FROM container c WHERE CONTAINS(c.name, "test")+ querySpan:+ type: 0+ results:+ - name: ""+ sourcecolumns: []+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: true+ sourcecolumns: []+ predicatecolumns: []+ predicatepaths:+ - container.namediff --git a/go.mod b/go.modindex 4eddeb7a00..ebd9e70a3e 100644--- a/go.mod+++ b/go.mod@@ -298,7 +298,7 @@ require (github.com/aws/aws-sdk-go-v2/service/sts v1.41.8github.com/aws/smithy-go v1.24.2 // indirectgithub.com/beorn7/perks v1.0.1 // indirect- github.com/bytebase/parser v0.0.0-20260228025849-39c76b45c98a+ github.com/bytebase/parser v0.0.0-20260318084752-6c474df57697github.com/cespare/xxhash/v2 v2.3.0 // indirectgithub.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirectgithub.com/cockroachdb/cockroach-go/v2 v2.4.3diff --git a/go.sum b/go.sumindex 6d023a40e0..295c4c5abd 100644--- a/go.sum+++ b/go.sum@@ -255,8 +255,8 @@ github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=github.com/bytebase/omni v0.0.0-20260318035856-4f43b2c93e35 h1:1z0q0B8LS8bmuOd2C2qC6UJKZSVz6cxHt09kItsqb/Y=github.com/bytebase/omni v0.0.0-20260318035856-4f43b2c93e35/go.mod h1:sHnw/Do16VheYa+ThYDBzY98EbkwFwWBzxh2CQuOgD0=-github.com/bytebase/parser v0.0.0-20260228025849-39c76b45c98a h1:hTyZhniv0HzQYM2K6mG1XpOEKibsfLJ7jYhBf/1sXLc=-github.com/bytebase/parser v0.0.0-20260228025849-39c76b45c98a/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=+github.com/bytebase/parser v0.0.0-20260318084752-6c474df57697 h1:/WgqQxX/gLsHS1WOx1rs5M/8kE9ecQAsgnQmlSr7p3c=+github.com/bytebase/parser v0.0.0-20260318084752-6c474df57697/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=github.com/bytebase/pkcs8 v0.0.0-20240612095628-fcd0a7484c94 h1:6PNsuNmSqCuR8Z616uQLzDIvujWCFsKzeVc3ECmVubk=