SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
fix(metrics): type-match filter values against the labels JSON read (#12519)
Tushar Vatseb01617c15e7
diff --git a/pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.go b/pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.goindex f5a90a6963..b2d58dfbd3 100644--- a/pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.go+++ b/pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.go@@ -322,6 +322,38 @@ func TestStatementBuilder(t *testing.T) {},expectedErr: nil,},+ {+ name: "test_bool_label_filter",+ requestType: qbtypes.RequestTypeTimeSeries,+ query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{+ Signal: telemetrytypes.SignalMetrics,+ StepInterval: qbtypes.Step{Duration: 30 * time.Second},+ Aggregations: []qbtypes.MetricAggregation{+ {+ MetricName: "signoz_calls_total",+ Type: metrictypes.SumType,+ Temporality: metrictypes.Cumulative,+ TimeAggregation: metrictypes.TimeAggregationRate,+ SpaceAggregation: metrictypes.SpaceAggregationSum,+ },+ },+ Filter: &qbtypes.Filter{+ Expression: "success = true",+ },+ GroupBy: []qbtypes.GroupByKey{+ {+ TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{+ Name: "service.name",+ },+ },+ },+ },+ expected: qbtypes.Statement{+ Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",+ Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", true, "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},+ },+ expectedErr: nil,+ },}fm := metricstelemetryschema.NewFieldMapper()diff --git a/pkg/statementbuilder/metricsstatementbuilder/testdata/keys_map.json b/pkg/statementbuilder/metricsstatementbuilder/testdata/keys_map.jsonindex c3d9c273a6..b436c34f59 100644--- a/pkg/statementbuilder/metricsstatementbuilder/testdata/keys_map.json+++ b/pkg/statementbuilder/metricsstatementbuilder/testdata/keys_map.json@@ -31,6 +31,14 @@"signal": "metrics"}],+ "success": [+ {+ "name": "success",+ "fieldContext": "attribute",+ "fieldDataType": "bool",+ "signal": "metrics"+ }+ ],"materialized.key.name": [{"name": "materialized.key.name",diff --git a/pkg/telemetryschema/metricstelemetryschema/condition_builder.go b/pkg/telemetryschema/metricstelemetryschema/condition_builder.goindex 82909a5fcc..374bdb775f 100644--- a/pkg/telemetryschema/metricstelemetryschema/condition_builder.go+++ b/pkg/telemetryschema/metricstelemetryschema/condition_builder.go@@ -5,6 +5,7 @@ import ("fmt""slices"+ schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator""github.com/SigNoz/signoz/pkg/errors""github.com/SigNoz/signoz/pkg/querybuilder"qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"@@ -22,6 +23,28 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {return &conditionBuilder{fm: fm}}+// Labels read back as String from the `labels` JSON whatever type the metadata claims, so the+// collision is always String vs the literal; intrinsic columns keep their own type.+func resolveTypeCollisionForFieldName(fieldExpression string, value any) string {+ if col, isColumn := timeSeriesV4Columns[fieldExpression]; isColumn {+ columnType := col.Type.GetType()+ if lowCardinality, ok := col.Type.(schema.LowCardinalityColumnType); ok {+ columnType = lowCardinality.ElementType.GetType()+ }+ if columnType != schema.ColumnTypeEnumString {+ return fieldExpression+ }+ }++ switch value.(type) {+ case bool:+ return fmt.Sprintf("accurateCastOrNull(%s, 'Bool')", fieldExpression)+ case float64:+ return fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)+ }+ return fieldExpression+}+func (c *conditionBuilder) conditionFor(ctx context.Context,orgID valuer.UUID,@@ -42,17 +65,8 @@ func (c *conditionBuilder) conditionFor(return "", err}- // TODO(srikanthccv): use the same data type collision handling when metrics schemas are updated- switch v := value.(type) {- case float64:- fieldExpression = fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)- case []any:- if len(v) > 0 && (operator == qbtypes.FilterOperatorBetween || operator == qbtypes.FilterOperatorNotBetween) {- if _, ok := v[0].(float64); ok {- fieldExpression = fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)- }- }- }+ // TODO(srikanthccv): use querybuilder.DataTypeCollisionHandledFieldName when metrics schemas are updated+ fieldExpression = resolveTypeCollisionForFieldName(fieldExpression, value)switch operator {case qbtypes.FilterOperatorEqual:@@ -100,6 +114,8 @@ func (c *conditionBuilder) conditionFor(if len(values) != 2 {return "", qbtypes.ErrBetweenValues}+ // both bounds share one expression, so the lower bound picks the cast+ fieldExpression = resolveTypeCollisionForFieldName(fieldExpression, values[0])return sb.Between(fieldExpression, values[0], values[1]), nilcase qbtypes.FilterOperatorNotBetween:values, ok := value.([]any)@@ -109,6 +125,7 @@ func (c *conditionBuilder) conditionFor(if len(values) != 2 {return "", qbtypes.ErrBetweenValues}+ fieldExpression = resolveTypeCollisionForFieldName(fieldExpression, values[0])return sb.NotBetween(fieldExpression, values[0], values[1]), nil// in and not in@@ -117,13 +134,23 @@ func (c *conditionBuilder) conditionFor(if !ok {return "", qbtypes.ErrInValues}- return sb.In(fieldExpression, values), nil+ // instead of using IN, we use `=` + `OR` to make use of index+ conditions := []string{}+ for _, item := range values {+ conditions = append(conditions, sb.E(resolveTypeCollisionForFieldName(fieldExpression, item), item))+ }+ return sb.Or(conditions...), nilcase qbtypes.FilterOperatorNotIn:values, ok := value.([]any)if !ok {return "", qbtypes.ErrInValues}- return sb.NotIn(fieldExpression, values), nil+ // instead of using NOT IN, we use `!=` + `AND` to make use of index+ conditions := []string{}+ for _, item := range values {+ conditions = append(conditions, sb.NE(resolveTypeCollisionForFieldName(fieldExpression, item), item))+ }+ return sb.And(conditions...), nil// exists and not exists// in the UI based query builder, `exists` and `not exists` are used fordiff --git a/pkg/telemetryschema/metricstelemetryschema/condition_builder_test.go b/pkg/telemetryschema/metricstelemetryschema/condition_builder_test.goindex 71dee826fa..44b1d1c624 100644--- a/pkg/telemetryschema/metricstelemetryschema/condition_builder_test.go+++ b/pkg/telemetryschema/metricstelemetryschema/condition_builder_test.go@@ -119,8 +119,8 @@ func TestConditionFor(t *testing.T) {},operator: qbtypes.FilterOperatorIn,value: []any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"},- expectedSQL: "metric_name IN (?)",- expectedArgs: []any{[]any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"}},+ expectedSQL: "(metric_name = ? OR metric_name = ? OR metric_name = ?)",+ expectedArgs: []any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"},expectedError: nil,},{@@ -155,8 +155,8 @@ func TestConditionFor(t *testing.T) {},operator: qbtypes.FilterOperatorNotIn,value: []any{"debug", "info", "trace"},- expectedSQL: "metric_name NOT IN (?)",- expectedArgs: []any{[]any{"debug", "info", "trace"}},+ expectedSQL: "(metric_name <> ? AND metric_name <> ? AND metric_name <> ?)",+ expectedArgs: []any{"debug", "info", "trace"},expectedError: nil,},{@@ -227,6 +227,120 @@ func TestConditionFor(t *testing.T) {expectedSQL: "",expectedError: qbtypes.ErrColumnNotFound,},+ {+ name: "Equal operator - bool label casts the JSON read to Bool",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "success",+ FieldContext: telemetrytypes.FieldContextAttribute,+ FieldDataType: telemetrytypes.FieldDataTypeBool,+ },+ operator: qbtypes.FilterOperatorEqual,+ value: true,+ expectedSQL: "accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ?",+ expectedArgs: []any{true},+ expectedError: nil,+ },+ {+ name: "Not Equal operator - bool label casts the JSON read to Bool",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "success",+ FieldContext: telemetrytypes.FieldContextAttribute,+ FieldDataType: telemetrytypes.FieldDataTypeBool,+ },+ operator: qbtypes.FilterOperatorNotEqual,+ value: false,+ expectedSQL: "accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') <> ?",+ expectedArgs: []any{false},+ expectedError: nil,+ },+ {+ name: "Equal operator - bool value on a label the metadata calls a string",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "success",+ FieldContext: telemetrytypes.FieldContextAttribute,+ FieldDataType: telemetrytypes.FieldDataTypeString,+ },+ operator: qbtypes.FilterOperatorEqual,+ value: true,+ expectedSQL: "accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ?",+ expectedArgs: []any{true},+ expectedError: nil,+ },+ {+ name: "In operator - all-bool set casts the JSON read to Bool",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "success",+ FieldContext: telemetrytypes.FieldContextAttribute,+ FieldDataType: telemetrytypes.FieldDataTypeBool,+ },+ operator: qbtypes.FilterOperatorIn,+ value: []any{true, false},+ expectedSQL: "(accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ? OR accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ?)",+ expectedArgs: []any{true, false},+ expectedError: nil,+ },+ {+ name: "In operator - a mixed set casts each value on its own",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "success",+ FieldContext: telemetrytypes.FieldContextAttribute,+ FieldDataType: telemetrytypes.FieldDataTypeBool,+ },+ operator: qbtypes.FilterOperatorIn,+ value: []any{true, "maybe"},+ expectedSQL: "(accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ? OR JSONExtractString(labels, 'success') = ?)",+ expectedArgs: []any{true, "maybe"},+ expectedError: nil,+ },+ {+ name: "Greater Than operator - a numeric column is compared without a cast",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "unix_milli",+ FieldContext: telemetrytypes.FieldContextMetric,+ },+ operator: qbtypes.FilterOperatorGreaterThan,+ value: float64(1747947419000),+ expectedSQL: "unix_milli > ?",+ expectedArgs: []any{float64(1747947419000)},+ expectedError: nil,+ },+ {+ name: "Equal operator - the is_monotonic column is already Bool, no cast",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "is_monotonic",+ FieldContext: telemetrytypes.FieldContextMetric,+ },+ operator: qbtypes.FilterOperatorEqual,+ value: true,+ expectedSQL: "is_monotonic = ?",+ expectedArgs: []any{true},+ expectedError: nil,+ },+ {+ name: "Between operator - the bounds cast the JSON read to Float64",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "latency",+ FieldContext: telemetrytypes.FieldContextAttribute,+ FieldDataType: telemetrytypes.FieldDataTypeFloat64,+ },+ operator: qbtypes.FilterOperatorBetween,+ value: []any{float64(10), float64(20)},+ expectedSQL: "toFloat64OrNull(JSONExtractString(labels, 'latency')) BETWEEN ? AND ?",+ expectedArgs: []any{float64(10), float64(20)},+ expectedError: nil,+ },+ {+ name: "Between operator - a numeric column is compared without a cast",+ key: telemetrytypes.TelemetryFieldKey{+ Name: "unix_milli",+ FieldContext: telemetrytypes.FieldContextMetric,+ },+ operator: qbtypes.FilterOperatorBetween,+ value: []any{float64(1747947419000), float64(1747947429000)},+ expectedSQL: "unix_milli BETWEEN ? AND ?",+ expectedArgs: []any{float64(1747947419000), float64(1747947429000)},+ expectedError: nil,+ },}fm := NewFieldMapper()diff --git a/tests/integration/tests/queriermetrics/13_bool_label_filter.py b/tests/integration/tests/queriermetrics/13_bool_label_filter.pynew file mode 100644index 0000000000..d9257a06bd--- /dev/null+++ b/tests/integration/tests/queriermetrics/13_bool_label_filter.py@@ -0,0 +1,66 @@+from collections.abc import Callable+from datetime import UTC, datetime, timedelta+from http import HTTPStatus++from fixtures import querier, types+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD+from fixtures.metrics import Metrics++METRIC = "test.metric.boollabel"+++def test_metrics_filter_bool_label(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ now = datetime.now(tz=UTC).replace(second=0, microsecond=0)+ insert_metrics(+ [+ Metrics(+ metric_name=METRIC,+ labels=labels,+ timestamp=now - timedelta(seconds=1),+ temporality="Unspecified",+ type_="Gauge",+ is_monotonic=False,+ value=value,+ )+ for labels, value in [+ ({"success": "true"}, 30.0),+ ({"success": "false"}, 10.0),+ ({"success": "1"}, 5.0),+ ({"success": "maybe"}, 3.0),+ ({"region": "us"}, 7.0),+ ]+ ]+ )+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ # `true` selects "true" and "1"; `false` selects only "false". "maybe" and the series+ # carrying no `success` label cast to NULL, so they are in neither result.+ for expr, expected in [+ ("success = true", 35.0),+ ("success = false", 10.0),+ ("success != true", 10.0),+ ("success IN [true]", 35.0),+ ("success IN [true, false]", 45.0),+ ]:+ response = querier.make_scalar_query_request(+ signoz,+ token,+ now,+ [+ querier.build_scalar_query(+ name="A",+ signal="metrics",+ aggregations=[querier.build_metrics_aggregation(METRIC, "latest", "sum", "unspecified", reduce_to="last")],+ filter_expression=expr,+ )+ ],+ )+ assert response.status_code == HTTPStatus.OK, f"{expr}: {response.text}"+ data = querier.get_scalar_table_data(response.json())+ assert len(data) == 1, f"{expr}: {data}"+ assert data[0][-1] == expected, f"{expr}: {data}"