SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
refactor(metrics): alias group-by columns positionally like logs and traces (#12464)
Tushar Vatsa7fd14eac9cb
diff --git a/pkg/statementbuilder/meterstatementbuilder/statement_builder.go b/pkg/statementbuilder/meterstatementbuilder/statement_builder.goindex c509b26c5c..14edeccba2 100644--- a/pkg/statementbuilder/meterstatementbuilder/statement_builder.go+++ b/pkg/statementbuilder/meterstatementbuilder/statement_builder.go@@ -150,12 +150,12 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts",stepSec,))- for _, g := range query.GroupBy {+ for i, g := range query.GroupBy {col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys)if err != nil {return "", nil, err}- sb.SelectMore(col)+ sb.SelectMore(fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(col), metricsstatementbuilder.GroupByColumnAlias(i, g.Name)))}tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)@@ -199,7 +199,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(sb.Where(sb.ILike("temporality", query.Aggregations[0].Temporality.StringValue()))}sb.GroupBy("ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(metricsstatementbuilder.GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, nil@@ -239,12 +239,12 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(stepSec,))- for _, g := range query.GroupBy {+ for i, g := range query.GroupBy {col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys)if err != nil {return "", nil, err}- sb.SelectMore(col)+ sb.SelectMore(fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(col), metricsstatementbuilder.GroupByColumnAlias(i, g.Name)))}tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)@@ -292,7 +292,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(}sb.GroupBy("fingerprint", "ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(metricsstatementbuilder.GroupByAliases(query.GroupBy)...)sb.OrderBy("fingerprint", "ts")q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)@@ -317,12 +317,12 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts",stepSec,))- for _, g := range query.GroupBy {+ for i, g := range query.GroupBy {col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys)if err != nil {return "", nil, err}- baseSb.SelectMore(col)+ baseSb.SelectMore(fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(col), metricsstatementbuilder.GroupByColumnAlias(i, g.Name)))}tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)@@ -363,7 +363,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(baseSb.Where(baseSb.ILike("temporality", query.Aggregations[0].Temporality.StringValue()))}baseSb.GroupBy("fingerprint", "ts")- baseSb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ baseSb.GroupBy(metricsstatementbuilder.GroupByAliases(query.GroupBy)...)baseSb.OrderBy("fingerprint", "ts")innerQuery, innerArgs := baseSb.BuildWithFlavor(sqlbuilder.ClickHouse)@@ -372,8 +372,8 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(case metrictypes.TimeAggregationRate:wrapped := sqlbuilder.NewSelectBuilder()wrapped.Select("ts")- for _, g := range query.GroupBy {- wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ wrapped.SelectMore(fmt.Sprintf("`%s`", metricsstatementbuilder.GroupByColumnAlias(i, g.Name)))}wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", metricsstatementbuilder.RateTmpl))wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))@@ -383,8 +383,8 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(case metrictypes.TimeAggregationIncrease:wrapped := sqlbuilder.NewSelectBuilder()wrapped.Select("ts")- for _, g := range query.GroupBy {- wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ wrapped.SelectMore(fmt.Sprintf("`%s`", metricsstatementbuilder.GroupByColumnAlias(i, g.Name)))}wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", metricsstatementbuilder.IncreaseTmpl))wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))@@ -413,8 +413,8 @@ func (b *meterQueryStatementBuilder) buildSpatialAggregationCTE(sb := sqlbuilder.NewSelectBuilder()sb.Select("ts")- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", metricsstatementbuilder.GroupByColumnAlias(i, g.Name)))}sb.SelectMore(fmt.Sprintf("%s(per_series_value) AS value", query.Aggregations[0].SpaceAggregation.StringValue()))sb.From("__temporal_aggregation_cte")@@ -423,7 +423,7 @@ func (b *meterQueryStatementBuilder) buildSpatialAggregationCTE(sb.Where(sb.EQ("per_series_value", query.Aggregations[0].ValueFilter.Value))}sb.GroupBy("ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(metricsstatementbuilder.GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, nildiff --git a/pkg/statementbuilder/meterstatementbuilder/stmt_builder_test.go b/pkg/statementbuilder/meterstatementbuilder/stmt_builder_test.goindex 39f19eda80..22b3011bfb 100644--- a/pkg/statementbuilder/meterstatementbuilder/stmt_builder_test.go+++ b/pkg/statementbuilder/meterstatementbuilder/stmt_builder_test.go@@ -2,6 +2,8 @@ package meterstatementbuilderimport ("context"+ "fmt"+ "strings""testing""time"@@ -14,6 +16,7 @@ import ("github.com/SigNoz/signoz/pkg/types/telemetrytypes""github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest""github.com/SigNoz/signoz/pkg/valuer"+ "github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -53,7 +56,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `service.name`, max(value) AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",+ Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, max(value) AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",Args: []any{"signoz_calls_total", uint64(1747785600000), uint64(1747983420000), "cartservice", "cumulative", 0},},expectedErr: nil,@@ -86,7 +89,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `service.name`, sum(value)/86400 AS value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",+ Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, sum(value)/86400 AS value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",Args: []any{"signoz_calls_total", uint64(1747872000000), uint64(1747983420000), "cartservice", "delta"},},expectedErr: nil,@@ -119,7 +122,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `service.name`, sum(value)/86400 AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `service.name`, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",+ Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, sum(value)/86400 AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",Args: []any{"signoz_calls_total", uint64(1747872000000), uint64(1747983420000), "cartservice", "delta", 0},},expectedErr: nil,@@ -152,7 +155,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'host.name') AS `host.name`, avg(value) AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'host.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `host.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `host.name`, ts",+ Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'host.name') AS `__GROUP_BY_KEY_0_host.name`, avg(value) AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'host.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_host.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_host.name`, ts",Args: []any{"system.memory.usage", uint64(1747872000000), uint64(1747983420000), "big-data-node-1", "unspecified", 0},},expectedErr: nil,@@ -197,3 +200,58 @@ func TestStatementBuilder(t *testing.T) {})}}++func TestGroupByAliasAvoidsColumnCollision(t *testing.T) {+ fm := metricstelemetryschema.NewFieldMapper()+ cb := metricstelemetryschema.NewConditionBuilder(fm)+ mockMetadataStore := telemetrytypestest.NewMockMetadataStore()+ keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")+ require.NoError(t, err)+ mockMetadataStore.KeysMap = keys++ flagger := flaggertest.New(t)++ statementBuilder := NewMeterQueryStatementBuilder(+ instrumentationtest.New().ToProviderSettings(),+ mockMetadataStore,+ fm,+ cb,+ metricsstatementbuilder.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, fm, cb, flagger),+ )++ for _, groupBy := range []string{"ts", "value", "fingerprint", "service.name"} {+ t.Run(groupBy, func(t *testing.T) {+ stmt, err := statementBuilder.Build(+ context.Background(),+ valuer.UUID{},+ 1747947419000,+ 1747983448000,+ qbtypes.RequestTypeTimeSeries,+ qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{+ Signal: telemetrytypes.SignalMetrics,+ StepInterval: qbtypes.Step{Duration: 24 * time.Hour},+ Aggregations: []qbtypes.MetricAggregation{+ {+ MetricName: "signoz_calls_total",+ Type: metrictypes.SumType,+ Temporality: metrictypes.Cumulative,+ TimeAggregation: metrictypes.TimeAggregationRate,+ SpaceAggregation: metrictypes.SpaceAggregationSum,+ },+ },+ GroupBy: []qbtypes.GroupByKey{+ {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: groupBy}},+ },+ },+ nil,+ )+ require.NoError(t, err)++ assert.Contains(t, stmt.Query, fmt.Sprintf("`__GROUP_BY_KEY_0_%s`", groupBy))+ assert.NotContains(t, stmt.Query, fmt.Sprintf("`%s`", groupBy),+ "the group-by column must not be selected under the label's own name")+ assert.Equal(t, 1, strings.Count(stmt.Query, " AS ts,"),+ "the step bucket is the only column named ts")+ })+ }+}diff --git a/pkg/statementbuilder/metricsstatementbuilder/reduced_test.go b/pkg/statementbuilder/metricsstatementbuilder/reduced_test.goindex 8343fe9c77..64abf9cd92 100644--- a/pkg/statementbuilder/metricsstatementbuilder/reduced_test.go+++ b/pkg/statementbuilder/metricsstatementbuilder/reduced_test.go@@ -31,6 +31,15 @@ func reducedQuery(metric string, ty metrictypes.Type, temp metrictypes.Temporali}}+func withGroupBy(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation], names ...string) qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation] {+ for _, name := range names {+ query.GroupBy = append(query.GroupBy, qbtypes.GroupByKey{+ TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: name},+ })+ }+ return query+}+func TestReducedStatementBuilder(t *testing.T) {cases := []struct {name string@@ -53,6 +62,14 @@ func TestReducedStatementBuilder(t *testing.T) {Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746997200000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000)},},},+ {+ name: "gauge_avg_avg_group_by",+ query: withGroupBy(reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg), "service.name"),+ expected: qbtypes.Statement{+ Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `__GROUP_BY_KEY_0_service.name`, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT points.reduced_fingerprint AS fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `__GROUP_BY_KEY_0_service.name`, avg(`sum_last`) AS per_series_value, avg(`count_series`) AS per_series_weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points FINAL INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name`), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",+ Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746997200000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000)},+ },+ },{name: "gauge_min_min",query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMin, metrictypes.SpaceAggregationMin),@@ -109,6 +126,14 @@ func TestReducedStatementBuilder(t *testing.T) {Args: []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746997200000), uint64(1747172760000), "test.metric.bucket", uint64(1746999900000), uint64(1747172760000)},},},+ {+ name: "histogram_p99_group_by",+ query: withGroupBy(reducedQuery("test.metric.bucket", metrictypes.HistogramType, metrictypes.Cumulative, metrictypes.TimeAggregationUnspecified, metrictypes.SpaceAggregationPercentile99), "service.name"),+ expected: qbtypes.Statement{+ Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(`sum`) / 300 AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points FINAL INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",+ Args: []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746997200000), uint64(1747172760000), "test.metric.bucket", uint64(1746999900000), uint64(1747172760000)},+ },+ },{name: "summary_avg",query: reducedQuery("test.metric", metrictypes.SummaryType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),diff --git a/pkg/statementbuilder/metricsstatementbuilder/statement_builder.go b/pkg/statementbuilder/metricsstatementbuilder/statement_builder.goindex fb5a6d8a3a..d6f14e18c1 100644--- a/pkg/statementbuilder/metricsstatementbuilder/statement_builder.go+++ b/pkg/statementbuilder/metricsstatementbuilder/statement_builder.go@@ -158,49 +158,13 @@ func (b *StatementBuilder) buildPipelineStatement(cteArgs [][]any)- origSpaceAgg := query.Aggregations[0].SpaceAggregation- origTimeAgg := query.Aggregations[0].TimeAggregation- origGroupBy := slices.Clone(query.GroupBy)-+ cteQuery := queryif query.Aggregations[0].Type == metrictypes.HistogramType {- // add le in the group by if doesn't exist- leExists := false- for _, g := range query.GroupBy {- if g.Name == "le" {- leExists = true- break- }- }-- if leExists {- // if the user themselves adds `le`, then we remove it from the original group by- // this is to avoid preparing a query that returns `nan`s, see following query- // SELECT- // ts,- // le,- // histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.99) AS value- // FROM __spatial_aggregation_cte- // GROUP BY- // le,- // ts-- origGroupBy = slices.DeleteFunc(origGroupBy, func(k qbtypes.GroupByKey) bool { return k.Name == "le" })- } else {- query.GroupBy = append(query.GroupBy, qbtypes.GroupByKey{- TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "le"},- })- }-- // make the time aggregation rate and space aggregation sum- if query.Aggregations[0].SpaceAggregation.IsPercentile() {- query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationRate- } else {- query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationIncrease- }- query.Aggregations[0].SpaceAggregation = metrictypes.SpaceAggregationSum+ query.GroupBy = slices.DeleteFunc(slices.Clone(query.GroupBy), isHistogramBucket)+ cteQuery = histogramCTEQuery(query)}- agg := query.Aggregations[0]+ agg := cteQuery.Aggregations[0]// A reduced metric reads the raw buffer for recent short windows, and// samples_v4/agg (unioned with the reduced tables) otherwise. The buffer is@@ -217,13 +181,13 @@ func (b *StatementBuilder) buildPipelineStatement(var timeSeriesCTEArgs []anyvar err error- if timeSeriesCTE, timeSeriesCTEArgs, err = b.buildTimeSeriesCTE(ctx, orgID, tsStart, tsEnd, query, keys, variables, tsTable); err != nil {+ if timeSeriesCTE, timeSeriesCTEArgs, err = b.buildTimeSeriesCTE(ctx, orgID, tsStart, tsEnd, cteQuery, keys, variables, tsTable); err != nil {return nil, err}- if qbtypes.CanShortCircuitDelta(query.Aggregations[0]) {+ if qbtypes.CanShortCircuitDelta(agg) {// spatial_aggregation_cte directly for certain delta queries- if frag, args, err := b.buildTemporalAggDeltaFastPath(start, end, query, samplesTable, timeSeriesCTE, timeSeriesCTEArgs); err != nil {+ if frag, args, err := b.buildTemporalAggDeltaFastPath(start, end, cteQuery, samplesTable, timeSeriesCTE, timeSeriesCTEArgs); err != nil {return nil, err} else if frag != "" {cteFragments = append(cteFragments, frag)@@ -231,7 +195,7 @@ func (b *StatementBuilder) buildPipelineStatement(}} else {// temporal_aggregation_cte- if frag, args, err := b.buildTemporalAggregationCTE(ctx, start, end, query, keys, samplesTable, timeSeriesCTE, timeSeriesCTEArgs); err != nil {+ if frag, args, err := b.buildTemporalAggregationCTE(ctx, start, end, cteQuery, keys, samplesTable, timeSeriesCTE, timeSeriesCTEArgs); err != nil {return nil, err} else if frag != "" {cteFragments = append(cteFragments, frag)@@ -239,7 +203,7 @@ func (b *StatementBuilder) buildPipelineStatement(}// spatial_aggregation_cte- if frag, args := b.buildSpatialAggregationCTE(ctx, start, end, query, keys); frag != "" {+ if frag, args := b.buildSpatialAggregationCTE(ctx, start, end, cteQuery, keys); frag != "" {cteFragments = append(cteFragments, frag)cteArgs = append(cteArgs, args)}@@ -252,27 +216,22 @@ func (b *StatementBuilder) buildPipelineStatement(var tsArgs []any// time series rows are written on hour boundariestsStart := start - (start % metricstelemetryschema.OneHourInMilliseconds)- if tsCTE, tsArgs, err = b.buildReducedTimeSeriesCTE(ctx, orgID, tsStart, end, query, keys, variables); err != nil {+ if tsCTE, tsArgs, err = b.buildReducedTimeSeriesCTE(ctx, orgID, tsStart, end, cteQuery, keys, variables); err != nil {return nil, err}if qbtypes.CanShortCircuitReduced(agg) {// spatial_aggregation_cte directly, no per-series level- if spatialFrag, spatialArgs, ok := b.buildReducedSpatialAggFastPath(start, end, query, tsCTE, tsArgs); ok {+ if spatialFrag, spatialArgs, ok := b.buildReducedSpatialAggFastPath(start, end, cteQuery, tsCTE, tsArgs); ok {reducedFragments = []string{spatialFrag}reducedArgs = [][]any{spatialArgs}}- } else if temporalFrag, temporalArgs, ok := b.buildReducedTemporalAggregationCTE(start, end, query, tsCTE, tsArgs); ok {- spatialFrag, spatialArgs := b.buildReducedSpatialAggregationCTE(query)+ } else if temporalFrag, temporalArgs, ok := b.buildReducedTemporalAggregationCTE(start, end, cteQuery, tsCTE, tsArgs); ok {+ spatialFrag, spatialArgs := b.buildReducedSpatialAggregationCTE(cteQuery)reducedFragments = []string{temporalFrag, spatialFrag}reducedArgs = [][]any{temporalArgs, spatialArgs}}}- // reset the query to the original state- query.Aggregations[0].SpaceAggregation = origSpaceAgg- query.Aggregations[0].TimeAggregation = origTimeAgg- query.GroupBy = origGroupBy-mainStmt, err := b.BuildFinalSelect(cteFragments, cteArgs, query)if err != nil {return nil, err@@ -289,8 +248,8 @@ func (b *StatementBuilder) buildPipelineStatement(func unionStatements(main, reduced *qbtypes.Statement, query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) (*qbtypes.Statement, error) {orderBy := "ts"- for _, g := range query.GroupBy {- orderBy = fmt.Sprintf("`%s`, ", g.Name) + orderBy+ for i, g := range query.GroupBy {+ orderBy = fmt.Sprintf("`%s`, ", GroupByColumnAlias(i, g.Name)) + orderBy}q := fmt.Sprintf("SELECT * FROM (%s) UNION ALL SELECT * FROM (%s) ORDER BY %s SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",@@ -333,12 +292,12 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))sb.Select("fingerprint")- for _, g := range query.GroupBy {+ for i, g := range query.GroupBy {col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys)if err != nil {return "", nil, err}- sb.SelectMore(col)+ sb.SelectMore(fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(col), GroupByColumnAlias(i, g.Name)))}sb.Where(sb.In("metric_name", query.Aggregations[0].MetricName),@@ -350,7 +309,7 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(sb.AddWhereClause(preparedWhereClause.WhereClause)}sb.GroupBy("fingerprint")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)return fmt.Sprintf("(%s) AS filtered_time_series", q), args, nil@@ -378,8 +337,8 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(sb := sqlbuilder.NewSelectBuilder()sb.Select(fmt.Sprintf("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts", stepSec))- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}sb.SelectMore(fmt.Sprintf("%s AS value", metricstelemetryschema.ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))@@ -390,7 +349,7 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(sb.LT("unix_milli", end),)sb.GroupBy("ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, true@@ -415,8 +374,8 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(sb := sqlbuilder.NewSelectBuilder()sb.Select("points.reduced_fingerprint AS fingerprint")sb.SelectMore(fmt.Sprintf("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts", stepSec))- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}sb.SelectMore(fmt.Sprintf("%s AS per_series_value", metricstelemetryschema.ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))if weight != "" {@@ -432,7 +391,7 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(sb.LT("unix_milli", end),)sb.GroupBy("fingerprint", "ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, true@@ -453,13 +412,13 @@ func (b *StatementBuilder) buildReducedSpatialAggregationCTE(sb := sqlbuilder.NewSelectBuilder()sb.Select("ts")- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}sb.SelectMore(spatial + " AS value")sb.From("__temporal_aggregation_cte")sb.GroupBy("ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args@@ -480,8 +439,8 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts",stepSec,))- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(@@ -510,7 +469,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(sb.LT("unix_milli", end),)sb.GroupBy("ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, nil@@ -551,12 +510,12 @@ func (b *StatementBuilder) buildTimeSeriesCTE(sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, tsTable))sb.Select("fingerprint")- for _, g := range query.GroupBy {+ for i, g := range query.GroupBy {col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys)if err != nil {return "", nil, err}- sb.SelectMore(col)+ sb.SelectMore(fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(col), GroupByColumnAlias(i, g.Name)))}sb.Where(@@ -580,7 +539,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(}sb.GroupBy("fingerprint")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)return fmt.Sprintf("(%s) AS filtered_time_series", q), args, nil@@ -620,8 +579,8 @@ func (b *StatementBuilder) buildTemporalAggDelta("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts",stepSec,))- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)@@ -643,7 +602,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(sb.LT("unix_milli", end),)sb.GroupBy("fingerprint", "ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)sb.OrderBy("fingerprint", "ts")q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)@@ -666,8 +625,8 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts",stepSec,))- for _, g := range query.GroupBy {- baseSb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ baseSb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)@@ -684,7 +643,7 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(baseSb.LT("unix_milli", end),)baseSb.GroupBy("fingerprint", "ts")- baseSb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ baseSb.GroupBy(GroupByAliases(query.GroupBy)...)baseSb.OrderBy("fingerprint", "ts")innerQuery, innerArgs := baseSb.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)@@ -693,8 +652,8 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(case metrictypes.TimeAggregationRate:wrapped := sqlbuilder.NewSelectBuilder()wrapped.Select("ts")- for _, g := range query.GroupBy {- wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ wrapped.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", RateTmpl))wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))@@ -704,8 +663,8 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(case metrictypes.TimeAggregationIncrease:wrapped := sqlbuilder.NewSelectBuilder()wrapped.Select("ts")- for _, g := range query.GroupBy {- wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ wrapped.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", IncreaseTmpl))wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))@@ -731,8 +690,8 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts",stepSec,))- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}aggForDeltaTemporality, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, metrictypes.Delta, query.Aggregations[0].TimeAggregation)@@ -775,7 +734,7 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(sb.LT("unix_milli", end),)sb.GroupBy("fingerprint", "ts", "temporality")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)queryWithoutWindow, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)queryWithWindowAndOrder := queryWithoutWindow + " WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint ASC, ts ASC) ORDER BY ts"return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", queryWithWindowAndOrder), args, nil@@ -791,8 +750,8 @@ func (b *StatementBuilder) buildSpatialAggregationCTE(sb := sqlbuilder.NewSelectBuilder()sb.Select("ts")- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}sb.SelectMore(fmt.Sprintf("%s(per_series_value) AS value", query.Aggregations[0].SpaceAggregation.StringValue()))sb.From("__temporal_aggregation_cte")@@ -801,7 +760,7 @@ func (b *StatementBuilder) buildSpatialAggregationCTE(sb.Where(sb.EQ("per_series_value", query.Aggregations[0].ValueFilter.Value))}sb.GroupBy("ts")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args@@ -827,15 +786,15 @@ func (b *StatementBuilder) BuildFinalSelect(if metricType == metrictypes.HistogramType && spaceAgg.IsPercentile() {quantile := query.Aggregations[0].SpaceAggregation.Percentile()sb.Select("ts")- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}sb.SelectMore(fmt.Sprintf("histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), %.3f) AS value",quantile,))sb.From("__spatial_aggregation_cte")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)sb.GroupBy("ts")if query.Having != nil && query.Having.Expression != "" {rewriter := querybuilder.NewHavingExpressionRewriter()@@ -848,8 +807,8 @@ func (b *StatementBuilder) BuildFinalSelect(} else if metricType == metrictypes.HistogramType && spaceAgg == metrictypes.SpaceAggregationCount && query.Aggregations[0].ComparisonSpaceAggregationParam != nil {sb.Select("ts")- for _, g := range query.GroupBy {- sb.SelectMore(fmt.Sprintf("`%s`", g.Name))+ for i, g := range query.GroupBy {+ sb.SelectMore(fmt.Sprintf("`%s`", GroupByColumnAlias(i, g.Name)))}aggQuery, err := metricstelemetryschema.AggregationQueryForHistogramCountWithParams(query.Aggregations[0].ComparisonSpaceAggregationParam)@@ -860,7 +819,7 @@ func (b *StatementBuilder) BuildFinalSelect(sb.From("__spatial_aggregation_cte")- sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.GroupBy(GroupByAliases(query.GroupBy)...)sb.GroupBy("ts")if query.Having != nil && query.Having.Expression != "" {@@ -884,7 +843,7 @@ func (b *StatementBuilder) BuildFinalSelect(sb.Where(rewrittenExpr)}}- sb.OrderBy(querybuilder.GroupByKeys(query.GroupBy)...)+ sb.OrderBy(GroupByAliases(query.GroupBy)...)sb.OrderBy("ts")if metricType == metrictypes.HistogramType && spaceAgg == metrictypes.SpaceAggregationCount && query.Aggregations[0].ComparisonSpaceAggregationParam == nil {sb.OrderBy("toFloat64(le)")@@ -893,3 +852,38 @@ func (b *StatementBuilder) BuildFinalSelect(q, a := sb.BuildWithFlavor(sqlbuilder.ClickHouse)return &qbtypes.Statement{Query: combined + q, Args: append(args, a...)}, nil}++const histogramBucketKey = "le"++func isHistogramBucket(k qbtypes.GroupByKey) bool { return k.Name == histogramBucketKey }++func histogramCTEQuery(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation] {+ query.GroupBy = append(slices.Clone(query.GroupBy), qbtypes.GroupByKey{+ TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: histogramBucketKey},+ })++ query.Aggregations = slices.Clone(query.Aggregations)+ if query.Aggregations[0].SpaceAggregation.IsPercentile() {+ query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationRate+ } else {+ query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationIncrease+ }+ query.Aggregations[0].SpaceAggregation = metrictypes.SpaceAggregationSum++ return query+}++func GroupByColumnAlias(i int, name string) string {+ if name == histogramBucketKey {+ return histogramBucketKey+ }+ return fmt.Sprintf("__GROUP_BY_KEY_%d_%s", i, name)+}++func GroupByAliases(groupBy []qbtypes.GroupByKey) []string {+ aliases := make([]string, 0, len(groupBy))+ for i := range groupBy {+ aliases = append(aliases, fmt.Sprintf("`%s`", GroupByColumnAlias(i, groupBy[i].Name)))+ }+ return aliases+}diff --git a/pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.go b/pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.goindex 3996145dd7..62896a825f 100644--- a/pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.go+++ b/pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.go@@ -2,6 +2,8 @@ package metricsstatementbuilderimport ("context"+ "fmt"+ "strings""testing""time"@@ -13,6 +15,7 @@ import ("github.com/SigNoz/signoz/pkg/types/telemetrytypes""github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest""github.com/SigNoz/signoz/pkg/valuer"+ "github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -52,7 +55,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",+ Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", "cartservice", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},},expectedErr: nil,@@ -85,7 +88,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND (match(JSONExtractString(labels, 'materialized.key.name'), ?) OR JSONExtractString(labels, 'service.name') = ?) GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",+ Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND (match(JSONExtractString(labels, 'materialized.key.name'), ?) OR JSONExtractString(labels, 'service.name') = ?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", "cartservice", "cartservice", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},},expectedErr: nil,@@ -118,7 +121,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",+ Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "delta", "cartservice", "signoz_calls_total", uint64(1747947390000), uint64(1747983420000)},},expectedErr: nil,@@ -151,11 +154,78 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts ORDER BY `service.name`, ts",+ Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",Args: []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "cartservice", "signoz_latency", uint64(1747947390000), uint64(1747983420000)},},expectedErr: nil,},+ {+ name: "test_histogram_percentile_explicit_le",+ requestType: qbtypes.RequestTypeTimeSeries,+ query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{+ Signal: telemetrytypes.SignalMetrics,+ StepInterval: qbtypes.Step{Duration: 30 * time.Second},+ Aggregations: []qbtypes.MetricAggregation{+ {+ MetricName: "signoz_latency",+ Type: metrictypes.HistogramType,+ Temporality: metrictypes.Delta,+ TimeAggregation: metrictypes.TimeAggregationRate,+ SpaceAggregation: metrictypes.SpaceAggregationPercentile95,+ },+ },+ Filter: &qbtypes.Filter{+ Expression: "service.name = 'cartservice'",+ },+ Limit: 10,+ GroupBy: []qbtypes.GroupByKey{+ {+ TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{+ Name: "le",+ },+ },+ {+ TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{+ Name: "service.name",+ },+ },+ },+ },+ expected: qbtypes.Statement{+ Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",+ Args: []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "cartservice", "signoz_latency", uint64(1747947390000), uint64(1747983420000)},+ },+ expectedErr: nil,+ },+ {+ name: "test_histogram_count_no_param",+ requestType: qbtypes.RequestTypeTimeSeries,+ query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{+ Signal: telemetrytypes.SignalMetrics,+ StepInterval: qbtypes.Step{Duration: 30 * time.Second},+ Aggregations: []qbtypes.MetricAggregation{+ {+ MetricName: "signoz_latency",+ Type: metrictypes.HistogramType,+ Temporality: metrictypes.Delta,+ TimeAggregation: metrictypes.TimeAggregationRate,+ SpaceAggregation: metrictypes.SpaceAggregationCount,+ },+ },+ GroupBy: []qbtypes.GroupByKey{+ {+ TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{+ Name: "service.name",+ },+ },+ },+ },+ expected: qbtypes.Statement{+ Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value) AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts, toFloat64(le)",+ Args: []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "signoz_latency", uint64(1747947390000), uint64(1747983420000)},+ },+ expectedErr: nil,+ },{name: "test_gauge_avg_sum",requestType: qbtypes.RequestTypeTimeSeries,@@ -184,7 +254,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint, `host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `host.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `host.name`, ts",+ Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `__GROUP_BY_KEY_0_host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_host.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_host.name`, ts",Args: []any{"system.memory.usage", uint64(1747936800000), uint64(1747983420000), "unspecified", "big-data-node-1", "system.memory.usage", uint64(1747947390000), uint64(1747983420000), 0},},expectedErr: nil,@@ -214,7 +284,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts ORDER BY `service.name`, ts",+ Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",Args: []any{"http_server_duration_bucket", uint64(1747936800000), uint64(1747983420000), "cumulative", "http_server_duration_bucket", uint64(1747947360000), uint64(1747983420000), 0},},expectedErr: nil,@@ -246,7 +316,7 @@ func TestStatementBuilder(t *testing.T) {},},expected: qbtypes.Statement{- Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `k8s.statefulset.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `k8s.statefulset.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'k8s.statefulset.name') AS `k8s.statefulset.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'k8s.statefulset.name') = ? GROUP BY fingerprint, `k8s.statefulset.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `k8s.statefulset.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `k8s.statefulset.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `k8s.statefulset.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `k8s.statefulset.name`, ts",+ Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_k8s.statefulset.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_k8s.statefulset.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'k8s.statefulset.name') AS `__GROUP_BY_KEY_0_k8s.statefulset.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'k8s.statefulset.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_k8s.statefulset.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_k8s.statefulset.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_k8s.statefulset.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_k8s.statefulset.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_k8s.statefulset.name`, ts",Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", "my-statefulset", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},},expectedErr: nil,@@ -299,3 +369,59 @@ func TestStatementBuilder(t *testing.T) {})}}++func TestGroupByAliasAvoidsColumnCollision(t *testing.T) {+ fm := metricstelemetryschema.NewFieldMapper()+ cb := metricstelemetryschema.NewConditionBuilder(fm)+ mockMetadataStore := telemetrytypestest.NewMockMetadataStore()+ keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")+ require.NoError(t, err)+ mockMetadataStore.KeysMap = keys++ fl, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry())+ require.NoError(t, err)++ statementBuilder := NewMetricQueryStatementBuilder(+ instrumentationtest.New().ToProviderSettings(),+ mockMetadataStore,+ fm,+ cb,+ fl,+ )++ for _, groupBy := range []string{"ts", "value", "fingerprint", "service.name"} {+ t.Run(groupBy, func(t *testing.T) {+ stmt, err := statementBuilder.Build(+ context.Background(),+ valuer.UUID{},+ 1747947419000,+ 1747983448000,+ qbtypes.RequestTypeTimeSeries,+ qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{+ Signal: telemetrytypes.SignalMetrics,+ StepInterval: qbtypes.Step{Duration: 30 * time.Second},+ Aggregations: []qbtypes.MetricAggregation{+ {+ MetricName: "signoz_calls_total",+ Type: metrictypes.SumType,+ Temporality: metrictypes.Cumulative,+ TimeAggregation: metrictypes.TimeAggregationRate,+ SpaceAggregation: metrictypes.SpaceAggregationSum,+ },+ },+ GroupBy: []qbtypes.GroupByKey{+ {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: groupBy}},+ },+ },+ nil,+ )+ require.NoError(t, err)++ assert.Contains(t, stmt.Query, fmt.Sprintf("`__GROUP_BY_KEY_0_%s`", groupBy))+ assert.NotContains(t, stmt.Query, fmt.Sprintf("`%s`", groupBy),+ "the group-by column must not be selected under the label's own name")+ assert.Equal(t, 1, strings.Count(stmt.Query, " AS ts,"),+ "the step bucket is the only column named ts")+ })+ }+}diff --git a/pkg/telemetryschema/metricstelemetryschema/field_mapper.go b/pkg/telemetryschema/metricstelemetryschema/field_mapper.goindex 275112bb4c..abd50164be 100644--- a/pkg/telemetryschema/metricstelemetryschema/field_mapper.go+++ b/pkg/telemetryschema/metricstelemetryschema/field_mapper.go@@ -9,7 +9,6 @@ import (qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5""github.com/SigNoz/signoz/pkg/types/telemetrytypes""github.com/SigNoz/signoz/pkg/valuer"- "github.com/huandu/go-sqlbuilder")var (@@ -103,13 +102,7 @@ func (m *fieldMapper) ColumnExpressionFor(startNs, endNs uint64,field *telemetrytypes.TelemetryFieldKey,_ telemetrytypes.FieldDataType,- keys map[string][]*telemetrytypes.TelemetryFieldKey,+ _ map[string][]*telemetrytypes.TelemetryFieldKey,) (string, error) {-- fieldExpression, err := m.FieldFor(ctx, orgID, startNs, endNs, field)- if err != nil {- return "", err- }-- return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil+ return m.FieldFor(ctx, orgID, startNs, endNs, field)}diff --git a/tests/integration/tests/queriermetrics/04_histogram.py b/tests/integration/tests/queriermetrics/04_histogram.pyindex b44d55d5eb..47db28a0e3 100644--- a/tests/integration/tests/queriermetrics/04_histogram.py+++ b/tests/integration/tests/queriermetrics/04_histogram.py@@ -505,3 +505,45 @@ def test_histogram_percentile_for_delta_service(assert result_values[0]["value"] == zeroth_valueassert result_values[1]["value"] == first_valueassert result_values[-1]["value"] == last_value+++def test_histogram_percentile_explicit_le_group_by(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ """`le` is grouped by internally and then collapsed by histogramQuantile, so naming it+ explicitly must not change the result — not even ahead of another key, where dropping it+ from the output shifts the remaining group-by positions."""+ now = datetime.now(tz=UTC).replace(second=0, microsecond=0)+ start_ms = int((now - timedelta(minutes=65)).timestamp() * 1000)+ end_ms = int(now.timestamp() * 1000)+ metric_name = "test_explicit_le_bucket"++ metrics = Metrics.load_from_file(+ FILE,+ base_time=now - timedelta(minutes=60),+ metric_name_override=metric_name,+ )+ insert_metrics(metrics)++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)+ response = make_query_request(+ signoz,+ token,+ start_ms,+ end_ms,+ [+ build_builder_query("A", metric_name, "doesnotreallymatter", "p95", group_by=["le", "service"]),+ build_builder_query("B", metric_name, "doesnotreallymatter", "p95", group_by=["service"]),+ ],+ )+ assert response.status_code == HTTPStatus.OK, response.text++ data = response.json()+ with_le = get_all_series(data, "A")+ without_le = get_all_series(data, "B")++ assert [[label["key"]["name"] for label in series["labels"]] for series in with_le] == [["service"], ["service"]], f"le must be collapsed out of the output labels, got {[series['labels'] for series in with_le]}"+ assert {series["labels"][0]["value"]: sorted(series["values"], key=lambda v: v["timestamp"]) for series in with_le} == {series["labels"][0]["value"]: sorted(series["values"], key=lambda v: v["timestamp"]) for series in without_le}diff --git a/tests/integration/tests/queriermetrics/10_key_resolution.py b/tests/integration/tests/queriermetrics/10_key_resolution.pyindex 7afe29f35b..4cefd18fe6 100644--- a/tests/integration/tests/queriermetrics/10_key_resolution.py+++ b/tests/integration/tests/queriermetrics/10_key_resolution.py@@ -256,3 +256,46 @@ def test_metrics_full_text_filter_does_not_error(# the term matches no series, and metrics emits no key-not-found warning.assert querier.get_scalar_table_data(response.json()) == [], f"{expr}: {response.json()}"assert querier.get_all_warnings(response.json()) == [], f"{expr}: {querier.get_all_warnings(response.json())}"+++def test_metrics_group_by_label_named_value(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ """A label may be named after a column the query builds for itself. Grouping by a label+ named `value` must alias it apart from the `value` the aggregation produces — selecting+ both under that one name is rejected by ClickHouse."""+ now = datetime.now(tz=UTC).replace(second=0, microsecond=0)+ insert_metrics(+ [+ Metrics(+ metric_name=METRIC,+ labels={"value": tier},+ timestamp=now - timedelta(seconds=1),+ temporality="Unspecified",+ type_="Gauge",+ is_monotonic=False,+ value=value,+ )+ for tier, value in [("high", 30.0), ("low", 10.0)]+ ]+ )+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ response = querier.make_scalar_query_request(+ signoz,+ token,+ now,+ [+ querier.build_scalar_query(+ name="A",+ signal="metrics",+ aggregations=[querier.build_metrics_aggregation(METRIC, "latest", "sum", "unspecified", reduce_to="last")],+ group_by=[querier.build_group_by_field("value", "string", "attribute")],+ )+ ],+ )+ assert response.status_code == HTTPStatus.OK, response.text+ assert {row[0]: row[-1] for row in querier.get_scalar_table_data(response.json())} == {"high": 30.0, "low": 10.0}