Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

fix(querier): limit scalar metric series after reduceTo (#13083)

Abhi kumar5cb2ed97cfaa
diff --git a/pkg/querier/postprocess.go b/pkg/querier/postprocess.go
index 0cb370e481..2046edf604 100644
--- a/pkg/querier/postprocess.go
+++ b/pkg/querier/postprocess.go
@@ -232,7 +232,11 @@ func postProcessMetricQuery(
}
}
- result = q.applySeriesLimit(result, query.Limit, query.Order)
+ reducesToScalar := req.RequestType == qbtypes.RequestTypeScalar && config.ReduceTo != qbtypes.ReduceToUnknown
+
+ if !reducesToScalar {
+ result = q.applySeriesLimit(result, query.Limit, query.Order)
+ }
if len(query.Functions) > 0 {
step := query.StepInterval.Milliseconds()
@@ -241,9 +245,11 @@ func postProcessMetricQuery(
}
// Apply reduce to for scalar request type
- if req.RequestType == qbtypes.RequestTypeScalar {
- if len(query.Aggregations) > 0 && query.Aggregations[0].ReduceTo != qbtypes.ReduceToUnknown {
- result = q.applyMetricReduceTo(result, query.Aggregations[0].ReduceTo)
+ if reducesToScalar {
+ result = q.applyMetricReduceTo(result, config.ReduceTo)
+ result = q.applySeriesLimit(result, query.Limit, query.Order)
+ if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
+ result.Value = convertTimeSeriesDataToScalar(tsData, tsData.QueryName)
}
}
@@ -287,9 +293,6 @@ func (q *querier) applyMetricReduceTo(result *qbtypes.Result, reduceOp qbtypes.R
}
}
- scalarData := convertTimeSeriesDataToScalar(tsData, tsData.QueryName)
- result.Value = scalarData
-
return result
}
diff --git a/pkg/querier/postprocess_test.go b/pkg/querier/postprocess_test.go
index a21a6b0a17..b2690da550 100644
--- a/pkg/querier/postprocess_test.go
+++ b/pkg/querier/postprocess_test.go
@@ -150,6 +150,107 @@ func TestProcessScalarFormula_AppliesOrderAndLimit(t *testing.T) {
})
}
+func TestPostProcessMetricQuery_LimitRanksByReducedValue(t *testing.T) {
+ q := &querier{
+ logger: instrumentationtest.New().Logger(),
+ }
+
+ serviceKey := telemetrytypes.TelemetryFieldKey{Name: "service.name", FieldDataType: telemetrytypes.FieldDataTypeString}
+ series := func(service string, points ...float64) *qbtypes.TimeSeries {
+ values := make([]*qbtypes.TimeSeriesValue, 0, len(points))
+ for i, point := range points {
+ values = append(values, &qbtypes.TimeSeriesValue{Timestamp: int64(i) * 60000, Value: point})
+ }
+ return &qbtypes.TimeSeries{Labels: []*qbtypes.Label{{Key: serviceKey, Value: service}}, Values: values}
+ }
+ // avg: checkout 10, payment 8.8, auth 6, search 2
+ makeResult := func() *qbtypes.Result {
+ return &qbtypes.Result{Value: &qbtypes.TimeSeriesData{
+ QueryName: "A",
+ Aggregations: []*qbtypes.AggregationBucket{{Series: []*qbtypes.TimeSeries{
+ series("checkout", 10, 10, 10, 10, 10),
+ series("payment", 1, 1, 1, 1, 40),
+ series("auth", 0, 0, 0, 0, 30),
+ series("search", 2, 2, 2, 2, 2),
+ }}},
+ }}
+ }
+ orderByValue := func(direction qbtypes.OrderDirection) []qbtypes.OrderBy {
+ return []qbtypes.OrderBy{{
+ Key: qbtypes.OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: qbtypes.DefaultOrderByKey}},
+ Direction: direction,
+ }}
+ }
+
+ testCases := []struct {
+ name string
+ reduceTo qbtypes.ReduceTo
+ order []qbtypes.OrderBy
+ expected [][]any
+ }{
+ {
+ name: "Max_Desc_KeepsHighestPeaks",
+ reduceTo: qbtypes.ReduceToMax,
+ order: orderByValue(qbtypes.OrderDirectionDesc),
+ expected: [][]any{{"payment", 40.0}, {"auth", 30.0}},
+ },
+ {
+ name: "Last_Desc_KeepsHighestLastPoints",
+ reduceTo: qbtypes.ReduceToLast,
+ order: orderByValue(qbtypes.OrderDirectionDesc),
+ expected: [][]any{{"payment", 40.0}, {"auth", 30.0}},
+ },
+ {
+ name: "Min_Asc_KeepsLowestMinimums",
+ reduceTo: qbtypes.ReduceToMin,
+ order: orderByValue(qbtypes.OrderDirectionAsc),
+ expected: [][]any{{"auth", 0.0}, {"payment", 1.0}},
+ },
+ {
+ name: "Max_NoOrder_DefaultsToDesc",
+ reduceTo: qbtypes.ReduceToMax,
+ expected: [][]any{{"payment", 40.0}, {"auth", 30.0}},
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ query := qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+ Name: "A",
+ Aggregations: []qbtypes.MetricAggregation{{MetricName: "http_requests_total", ReduceTo: testCase.reduceTo}},
+ Order: testCase.order,
+ Limit: 2,
+ }
+ req := &qbtypes.QueryRangeRequest{RequestType: qbtypes.RequestTypeScalar}
+
+ out := postProcessMetricQuery(q, makeResult(), query, req)
+ scalar, ok := out.Value.(*qbtypes.ScalarData)
+ require.True(t, ok, "expected *ScalarData, got %T", out.Value)
+
+ assert.Equal(t, testCase.expected, scalar.Data)
+ })
+ }
+
+ t.Run("TimeSeries_RanksByAverage", func(t *testing.T) {
+ query := qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+ Name: "A",
+ Aggregations: []qbtypes.MetricAggregation{{MetricName: "http_requests_total"}},
+ Order: orderByValue(qbtypes.OrderDirectionDesc),
+ Limit: 2,
+ }
+ req := &qbtypes.QueryRangeRequest{RequestType: qbtypes.RequestTypeTimeSeries}
+
+ out := postProcessMetricQuery(q, makeResult(), query, req)
+ tsData, ok := out.Value.(*qbtypes.TimeSeriesData)
+ require.True(t, ok, "expected *TimeSeriesData, got %T", out.Value)
+ require.Len(t, tsData.Aggregations, 1)
+ require.Len(t, tsData.Aggregations[0].Series, 2)
+
+ assert.Equal(t, "checkout", tsData.Aggregations[0].Series[0].Labels[0].Value)
+ assert.Equal(t, "payment", tsData.Aggregations[0].Series[1].Labels[0].Value)
+ })
+}
+
// Multiple series with different number of labels, shouldn't panic and should align labels correctly.
func TestConvertTimeSeriesDataToScalar_RaggedLabels(t *testing.T) {
label := func(name string, value any) *qbtypes.Label {
diff --git a/tests/integration/tests/querierscalar/05_reduce_to.py b/tests/integration/tests/querierscalar/05_reduce_to.py
index be2152a803..a57f876c99 100644
--- a/tests/integration/tests/querierscalar/05_reduce_to.py
+++ b/tests/integration/tests/querierscalar/05_reduce_to.py
@@ -11,6 +11,7 @@ from fixtures.querier import (
assert_scalar_result_order,
build_group_by_field,
build_metrics_aggregation,
+ build_order_by,
build_scalar_query,
get_scalar_table_data,
make_scalar_query_request,
@@ -72,6 +73,125 @@ def test_metrics_scalar_reduce_to_modes(
assert_scalar_result_order(data, [("service-a", expected)], f"metrics reduceTo={reduce_to}")
+ "reduce_to,order,expected",
+ [
+ pytest.param("max", "desc", [("payment", 40.0), ("auth", 30.0)], id="max_desc"),
+ pytest.param("last", "desc", [("payment", 40.0), ("auth", 30.0)], id="last_desc"),
+ pytest.param("min", "asc", [("auth", 0.0), ("payment", 1.0)], id="min_asc"),
+ pytest.param("max", None, [("payment", 40.0), ("auth", 30.0)], id="max_no_order"),
+ ],
+)
+def test_metrics_scalar_limit_ranks_by_reduced_value(
+ signoz: types.SigNoz,
+ create_user_admin: None, # pylint: disable=unused-argument
+ get_token: Callable[[str, str], str],
+ insert_metrics: Callable[[list[Metrics]], None],
+ reduce_to: str,
+ order: str | None,
+ expected: list[tuple[str, float]],
+) -> None:
+ """Limit keeps the top series by the reduced value, not by their average
+ (avg: checkout 10, payment 8.8, auth 6, search 2)."""
+ now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+ metric_name = "test.reduce.limit.metric"
+ points = {
+ "checkout": [10.0, 10.0, 10.0, 10.0, 10.0],
+ "payment": [1.0, 1.0, 1.0, 1.0, 40.0],
+ "auth": [0.0, 0.0, 0.0, 0.0, 30.0],
+ "search": [2.0, 2.0, 2.0, 2.0, 2.0],
+ }
+ insert_metrics(
+ [
+ Metrics(
+ metric_name=metric_name,
+ labels={"service.name": service},
+ timestamp=now - timedelta(seconds=60 * (len(values) - 1 - i) + 1),
+ value=value,
+ temporality="Unspecified",
+ type_="Gauge",
+ is_monotonic=False,
+ )
+ for service, values in points.items()
+ for i, value in enumerate(values)
+ ]
+ )
+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+ query = build_scalar_query(
+ name="A",
+ signal="metrics",
+ aggregations=[build_metrics_aggregation(metric_name, "latest", "sum", "unspecified", reduce_to=reduce_to)],
+ group_by=[build_group_by_field("service.name", "string", "attribute")],
+ order=[build_order_by(f"sum({metric_name})", order)] if order else None,
+ limit=2,
+ )
+ response = make_scalar_query_request(signoz, token, now, [query])
+
+ assert response.status_code == HTTPStatus.OK, response.text
+ assert response.json()["status"] == "success"
+ data = get_scalar_table_data(response.json())
+ assert_scalar_result_order(data, expected, f"metrics reduceTo={reduce_to} order={order} limit=2")
+
+
+ "functions,expected",
+ [
+ pytest.param([{"name": "clampMax", "args": [{"value": 35}]}], [("payment", 35.0), ("auth", 30.0)], id="clamp_max"),
+ pytest.param([{"name": "runningDiff"}], [("payment", 39.0), ("auth", 30.0)], id="running_diff"),
+ ],
+)
+def test_metrics_scalar_limit_ranks_after_functions(
+ signoz: types.SigNoz,
+ create_user_admin: None, # pylint: disable=unused-argument
+ get_token: Callable[[str, str], str],
+ insert_metrics: Callable[[list[Metrics]], None],
+ functions: list[dict],
+ expected: list[tuple[str, float]],
+) -> None:
+ """Functions run before reduceTo (max) and limit, so ranking uses the function output."""
+ now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+ metric_name = "test.reduce.limit.functions.metric"
+ points = {
+ "checkout": [10.0, 10.0, 10.0, 10.0, 10.0],
+ "payment": [1.0, 1.0, 1.0, 1.0, 40.0],
+ "auth": [0.0, 0.0, 0.0, 0.0, 30.0],
+ "search": [2.0, 2.0, 2.0, 2.0, 2.0],
+ }
+ insert_metrics(
+ [
+ Metrics(
+ metric_name=metric_name,
+ labels={"service.name": service},
+ timestamp=now - timedelta(seconds=60 * (len(values) - 1 - i) + 1),
+ value=value,
+ temporality="Unspecified",
+ type_="Gauge",
+ is_monotonic=False,
+ )
+ for service, values in points.items()
+ for i, value in enumerate(values)
+ ]
+ )
+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+ query = build_scalar_query(
+ name="A",
+ signal="metrics",
+ aggregations=[build_metrics_aggregation(metric_name, "latest", "sum", "unspecified", reduce_to="max")],
+ group_by=[build_group_by_field("service.name", "string", "attribute")],
+ order=[build_order_by(f"sum({metric_name})", "desc")],
+ limit=2,
+ functions=functions,
+ )
+ response = make_scalar_query_request(signoz, token, now, [query])
+
+ assert response.status_code == HTTPStatus.OK, response.text
+ assert response.json()["status"] == "success"
+ data = get_scalar_table_data(response.json())
+ assert_scalar_result_order(data, expected, f"metrics functions={functions} reduceTo=max limit=2")
+
+
def test_metrics_scalar_missing_reduce_to_rejected(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument