Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tracing): add endpoints mcp tool (#125402)

Jon McCallumda07decf282a
diff --git a/products/tracing/backend/presentation/views.py b/products/tracing/backend/presentation/views.py
index 38cc7afdeef7..1224a4172a21 100644
--- a/products/tracing/backend/presentation/views.py
+++ b/products/tracing/backend/presentation/views.py
@@ -21,7 +21,11 @@ from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_
from pydantic import ValidationError
from rest_framework import serializers, status, viewsets
from rest_framework.decorators import action
-from rest_framework.exceptions import ParseError, PermissionDenied
+from rest_framework.exceptions import (
+ ParseError,
+ PermissionDenied,
+ ValidationError as DRFValidationError,
+)
from rest_framework.request import Request
from rest_framework.response import Response
@@ -618,6 +622,15 @@ class _TracingAggregationQueryBodySerializer(serializers.Serializer):
"and resource attribute maps, which the rest of the aggregation never touches."
),
)
+ groupByEndpoint = serializers.BooleanField(
+ required=False,
+ default=False,
+ help_text=(
+ "Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths "
+ "like `/users/1` and `/users/2` fold into one row. Spans without a route keep their span name. Spans "
+ "that are not HTTP server spans are left out."
+ ),
+ )
class _TracingAggregationRequestSerializer(serializers.Serializer):
@@ -1217,6 +1230,16 @@ class SpansViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)
return default
return max(minimum, parsed)
+ @staticmethod
+ def _parse_bool(value: object) -> bool:
+ """Parse an untrusted JSON flag the way the declared BooleanField would, so `"false"` stays false."""
+ if value is None:
+ return False
+ try:
+ return serializers.BooleanField().run_validation(value)
+ except DRFValidationError as e:
+ raise ParseError(f"Expected a boolean, got {value!r}.") from e
+
def _report_usage(self, request: Request, event: str, properties: dict) -> None:
# Usage telemetry must never turn a successful read into a 5xx, so swallow and record any failure.
try:
@@ -1711,6 +1734,7 @@ class SpansViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)
limit=min(limit + 1, _ROW_LIMIT),
offset=offset,
include_impact=bool(query_data.get("includeImpact")),
+ group_by_endpoint=self._parse_bool(query_data.get("groupByEndpoint")),
)
results = list(response.results)
@@ -1732,6 +1756,7 @@ class SpansViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)
"has_filter_group": bool(query_data.get("filterGroup")),
"service_names_count": len(query_data.get("serviceNames") or []),
"include_impact": bool(query_data.get("includeImpact")),
+ "group_by_endpoint": bool(query_data.get("groupByEndpoint")),
},
)
diff --git a/products/tracing/backend/tests/test_aggregation_endpoint.py b/products/tracing/backend/tests/test_aggregation_endpoint.py
index 25547a2c107a..4ab8bb23b1e8 100644
--- a/products/tracing/backend/tests/test_aggregation_endpoint.py
+++ b/products/tracing/backend/tests/test_aggregation_endpoint.py
@@ -67,3 +67,17 @@ class TestTraceSpansAggregationEndpoint(_TraceSpansTestBase):
expected_more = offset + page_size < self.TOTAL_NAMES
self.assertEqual(data["has_more"], expected_more)
self.assertEqual(data["next_offset"], offset + page_size if expected_more else None)
+
+ @parameterized.expand([("bool_true", True, True), ("string_true", "true", True), ("string_false", "false", False)])
+ def test_group_by_endpoint_reaches_the_query(self, _name, value, grouped):
+ # The fixture spans carry no HTTP method, so endpoint grouping scopes every one of them out.
+ data = self._aggregate(groupByEndpoint=value)
+ self.assertEqual(data["results"] == [], grouped)
+
+ def test_group_by_endpoint_rejects_a_non_boolean(self):
+ response = self.client.post(
+ f"/api/projects/{self.team.id}/tracing/spans/aggregate/",
+ {"query": {"dateRange": {"date_from": DATE_FROM, "date_to": DATE_TO}, "groupByEndpoint": "maybe"}},
+ format="json",
+ )
+ self.assertEqual(response.status_code, 400, response.content)
diff --git a/products/tracing/frontend/generated/api.schemas.ts b/products/tracing/frontend/generated/api.schemas.ts
index 9056085cfc5b..94c34ca37d2a 100644
--- a/products/tracing/frontend/generated/api.schemas.ts
+++ b/products/tracing/frontend/generated/api.schemas.ts
@@ -247,6 +247,8 @@ export interface _TracingAggregationQueryBodyApi {
offset?: number
/** Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches. */
includeImpact?: boolean
+ /** Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `/users/1` and `/users/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out. */
+ groupByEndpoint?: boolean
}
export interface _TracingAggregationRequestApi {
diff --git a/products/tracing/frontend/generated/api.zod.ts b/products/tracing/frontend/generated/api.zod.ts
index d9d9f739e46c..c84b0d8a1372 100644
--- a/products/tracing/frontend/generated/api.zod.ts
+++ b/products/tracing/frontend/generated/api.zod.ts
@@ -174,6 +174,7 @@ export const tracingSpansAggregateCreateBodyQueryOneLimitMax = 5000
export const tracingSpansAggregateCreateBodyQueryOneOffsetMin = 0
export const tracingSpansAggregateCreateBodyQueryOneIncludeImpactDefault = false
+export const tracingSpansAggregateCreateBodyQueryOneGroupByEndpointDefault = false
export const TracingSpansAggregateCreateBody = /* @__PURE__ */ zod.object({
query: zod
@@ -283,6 +284,12 @@ export const TracingSpansAggregateCreateBody = /* @__PURE__ */ zod.object({
.describe(
'Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches.'
),
+ groupByEndpoint: zod
+ .boolean()
+ .default(tracingSpansAggregateCreateBodyQueryOneGroupByEndpointDefault)
+ .describe(
+ 'Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `\/users\/1` and `\/users\/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out.'
+ ),
})
.describe('The span aggregation query to execute.'),
})
diff --git a/products/tracing/mcp/prompts/apm-spans-aggregate.md b/products/tracing/mcp/prompts/apm-spans-aggregate.md
index ba00dc5ccf7a..f7cbc8ebdf4e 100644
--- a/products/tracing/mcp/prompts/apm-spans-aggregate.md
+++ b/products/tracing/mcp/prompts/apm-spans-aggregate.md
@@ -90,6 +90,14 @@ Max rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard
Row offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.
+## query.groupByEndpoint
+
+Set to `true` to get per-endpoint health for HTTP servers. Rows then group by `METHOD route` (for example `GET /users/{id}`) from the `http.route` attribute, in place of the span name.
+
+- Only HTTP server spans (span kind server with an HTTP method) are counted. Queue consumers, workflow activities and client spans are left out.
+- A span without `http.route` keeps its span name as the row key.
+- Use it when span names are raw paths (`/users/1`, `/users/2`) or a bare method (`GET`), which split or merge routes.
+
# Examples
## Top operations by total time in the last hour
@@ -146,6 +154,20 @@ Returns `results` for the last hour and `compare` for the hour before. Diff `tot
}
```
+## Endpoint health for one service
+
+```json
+{
+ "query": {
+ "serviceNames": ["api-gateway"],
+ "groupByEndpoint": true,
+ "dateRange": { "date_from": "-1d" }
+ }
+}
+```
+
+Each row is one route. Compare `error_count / count` and `p95_duration_nano` across rows to find unhealthy endpoints.
+
## Aggregate within one service
```json
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 2a5ac0bf9038..577eecf9d37e 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -1242,7 +1242,7 @@
"feature_flag": "tracing"
},
"apm-spans-aggregate": {
- "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.",
+ "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n## query.groupByEndpoint\n\nSet to `true` to get per-endpoint health for HTTP servers. Rows then group by `METHOD route` (for example `GET /users/{id}`) from the `http.route` attribute, in place of the span name.\n\n- Only HTTP server spans (span kind server with an HTTP method) are counted. Queue consumers, workflow activities and client spans are left out.\n- A span without `http.route` keeps its span name as the row key.\n- Use it when span names are raw paths (`/users/1`, `/users/2`) or a bare method (`GET`), which split or merge routes.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Endpoint health for one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"groupByEndpoint\": true,\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nEach row is one route. Compare `error_count / count` and `p95_duration_nano` across rows to find unhealthy endpoints.\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.",
"category": "Tracing",
"feature": "tracing",
"summary": "Aggregate span statistics",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index 6ee512936214..9ffffc576288 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -1257,7 +1257,7 @@
"feature_flag": "tracing"
},
"apm-spans-aggregate": {
- "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.",
+ "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n## query.groupByEndpoint\n\nSet to `true` to get per-endpoint health for HTTP servers. Rows then group by `METHOD route` (for example `GET /users/{id}`) from the `http.route` attribute, in place of the span name.\n\n- Only HTTP server spans (span kind server with an HTTP method) are counted. Queue consumers, workflow activities and client spans are left out.\n- A span without `http.route` keeps its span name as the row key.\n- Use it when span names are raw paths (`/users/1`, `/users/2`) or a bare method (`GET`), which split or merge routes.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Endpoint health for one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"groupByEndpoint\": true,\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nEach row is one route. Compare `error_count / count` and `p95_duration_nano` across rows to find unhealthy endpoints.\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.",
"category": "Tracing",
"feature": "tracing",
"summary": "Aggregate span statistics",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index e9b540aaa761..bb3681f0e9dc 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -127306,6 +127306,8 @@ export namespace Schemas {
offset?: number;
/** Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches. */
includeImpact?: boolean;
+ /** Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `/users/1` and `/users/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out. */
+ groupByEndpoint?: boolean;
}
export interface _TracingAggregationRequest {
diff --git a/services/mcp/src/generated/tracing/api.ts b/services/mcp/src/generated/tracing/api.ts
index 73ba61a02e02..a332838fd08c 100644
--- a/services/mcp/src/generated/tracing/api.ts
+++ b/services/mcp/src/generated/tracing/api.ts
@@ -23,6 +23,7 @@ export const tracingSpansAggregateCreateBodyQueryOneLimitMax = 5000
export const tracingSpansAggregateCreateBodyQueryOneOffsetMin = 0
export const tracingSpansAggregateCreateBodyQueryOneIncludeImpactDefault = false
+export const tracingSpansAggregateCreateBodyQueryOneGroupByEndpointDefault = false
export const TracingSpansAggregateCreateBody = () => zod.object({
query: zod
@@ -132,6 +133,12 @@ export const TracingSpansAggregateCreateBody = () => zod.object({
.describe(
'Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches.'
),
+ groupByEndpoint: zod
+ .boolean()
+ .default(tracingSpansAggregateCreateBodyQueryOneGroupByEndpointDefault)
+ .describe(
+ 'Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `\/users\/1` and `\/users\/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out.'
+ ),
})
.describe('The span aggregation query to execute.'),
})
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/apm-spans-aggregate.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/apm-spans-aggregate.json
index 51988f17bd9a..cead78dd5995 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/apm-spans-aggregate.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/apm-spans-aggregate.json
@@ -97,6 +97,11 @@
},
"type": "array"
},
+ "groupByEndpoint": {
+ "default": false,
+ "description": "Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `/users/1` and `/users/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out.",
+ "type": "boolean"
+ },
"includeImpact": {
"default": false,
"description": "Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches.",