:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tracing): add endpoints mcp tool (#125402)
Jon McCallumda07decf282a
diff --git a/products/tracing/backend/presentation/views.py b/products/tracing/backend/presentation/views.pyindex 38cc7afdeef7..1224a4172a21 100644--- a/products/tracing/backend/presentation/views.py+++ b/products/tracing/backend/presentation/views.py@@ -21,7 +21,11 @@ from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_from pydantic import ValidationErrorfrom rest_framework import serializers, status, viewsetsfrom rest_framework.decorators import action-from rest_framework.exceptions import ParseError, PermissionDenied+from rest_framework.exceptions import (+ ParseError,+ PermissionDenied,+ ValidationError as DRFValidationError,+)from rest_framework.request import Requestfrom rest_framework.response import Response@@ -618,6 +622,15 @@ class _TracingAggregationQueryBodySerializer(serializers.Serializer):"and resource attribute maps, which the rest of the aggregation never touches."),)+ groupByEndpoint = serializers.BooleanField(+ required=False,+ default=False,+ help_text=(+ "Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths "+ "like `/users/1` and `/users/2` fold into one row. Spans without a route keep their span name. Spans "+ "that are not HTTP server spans are left out."+ ),+ )class _TracingAggregationRequestSerializer(serializers.Serializer):@@ -1217,6 +1230,16 @@ class SpansViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)return defaultreturn max(minimum, parsed)+ @staticmethod+ def _parse_bool(value: object) -> bool:+ """Parse an untrusted JSON flag the way the declared BooleanField would, so `"false"` stays false."""+ if value is None:+ return False+ try:+ return serializers.BooleanField().run_validation(value)+ except DRFValidationError as e:+ raise ParseError(f"Expected a boolean, got {value!r}.") from e+def _report_usage(self, request: Request, event: str, properties: dict) -> None:# Usage telemetry must never turn a successful read into a 5xx, so swallow and record any failure.try:@@ -1711,6 +1734,7 @@ class SpansViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)limit=min(limit + 1, _ROW_LIMIT),offset=offset,include_impact=bool(query_data.get("includeImpact")),+ group_by_endpoint=self._parse_bool(query_data.get("groupByEndpoint")),)results = list(response.results)@@ -1732,6 +1756,7 @@ class SpansViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)"has_filter_group": bool(query_data.get("filterGroup")),"service_names_count": len(query_data.get("serviceNames") or []),"include_impact": bool(query_data.get("includeImpact")),+ "group_by_endpoint": bool(query_data.get("groupByEndpoint")),},)diff --git a/products/tracing/backend/tests/test_aggregation_endpoint.py b/products/tracing/backend/tests/test_aggregation_endpoint.pyindex 25547a2c107a..4ab8bb23b1e8 100644--- a/products/tracing/backend/tests/test_aggregation_endpoint.py+++ b/products/tracing/backend/tests/test_aggregation_endpoint.py@@ -67,3 +67,17 @@ class TestTraceSpansAggregationEndpoint(_TraceSpansTestBase):expected_more = offset + page_size < self.TOTAL_NAMESself.assertEqual(data["has_more"], expected_more)self.assertEqual(data["next_offset"], offset + page_size if expected_more else None)++ @parameterized.expand([("bool_true", True, True), ("string_true", "true", True), ("string_false", "false", False)])+ def test_group_by_endpoint_reaches_the_query(self, _name, value, grouped):+ # The fixture spans carry no HTTP method, so endpoint grouping scopes every one of them out.+ data = self._aggregate(groupByEndpoint=value)+ self.assertEqual(data["results"] == [], grouped)++ def test_group_by_endpoint_rejects_a_non_boolean(self):+ response = self.client.post(+ f"/api/projects/{self.team.id}/tracing/spans/aggregate/",+ {"query": {"dateRange": {"date_from": DATE_FROM, "date_to": DATE_TO}, "groupByEndpoint": "maybe"}},+ format="json",+ )+ self.assertEqual(response.status_code, 400, response.content)diff --git a/products/tracing/frontend/generated/api.schemas.ts b/products/tracing/frontend/generated/api.schemas.tsindex 9056085cfc5b..94c34ca37d2a 100644--- a/products/tracing/frontend/generated/api.schemas.ts+++ b/products/tracing/frontend/generated/api.schemas.ts@@ -247,6 +247,8 @@ export interface _TracingAggregationQueryBodyApi {offset?: number/** Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches. */includeImpact?: boolean+ /** Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `/users/1` and `/users/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out. */+ groupByEndpoint?: boolean}export interface _TracingAggregationRequestApi {diff --git a/products/tracing/frontend/generated/api.zod.ts b/products/tracing/frontend/generated/api.zod.tsindex d9d9f739e46c..c84b0d8a1372 100644--- a/products/tracing/frontend/generated/api.zod.ts+++ b/products/tracing/frontend/generated/api.zod.ts@@ -174,6 +174,7 @@ export const tracingSpansAggregateCreateBodyQueryOneLimitMax = 5000export const tracingSpansAggregateCreateBodyQueryOneOffsetMin = 0export const tracingSpansAggregateCreateBodyQueryOneIncludeImpactDefault = false+export const tracingSpansAggregateCreateBodyQueryOneGroupByEndpointDefault = falseexport const TracingSpansAggregateCreateBody = /* @__PURE__ */ zod.object({query: zod@@ -283,6 +284,12 @@ export const TracingSpansAggregateCreateBody = /* @__PURE__ */ zod.object({.describe('Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches.'),+ groupByEndpoint: zod+ .boolean()+ .default(tracingSpansAggregateCreateBodyQueryOneGroupByEndpointDefault)+ .describe(+ 'Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `\/users\/1` and `\/users\/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out.'+ ),}).describe('The span aggregation query to execute.'),})diff --git a/products/tracing/mcp/prompts/apm-spans-aggregate.md b/products/tracing/mcp/prompts/apm-spans-aggregate.mdindex ba00dc5ccf7a..f7cbc8ebdf4e 100644--- a/products/tracing/mcp/prompts/apm-spans-aggregate.md+++ b/products/tracing/mcp/prompts/apm-spans-aggregate.md@@ -90,6 +90,14 @@ Max rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hardRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.+## query.groupByEndpoint++Set to `true` to get per-endpoint health for HTTP servers. Rows then group by `METHOD route` (for example `GET /users/{id}`) from the `http.route` attribute, in place of the span name.++- Only HTTP server spans (span kind server with an HTTP method) are counted. Queue consumers, workflow activities and client spans are left out.+- A span without `http.route` keeps its span name as the row key.+- Use it when span names are raw paths (`/users/1`, `/users/2`) or a bare method (`GET`), which split or merge routes.+# Examples## Top operations by total time in the last hour@@ -146,6 +154,20 @@ Returns `results` for the last hour and `compare` for the hour before. Diff `tot}```+## Endpoint health for one service++```json+{+ "query": {+ "serviceNames": ["api-gateway"],+ "groupByEndpoint": true,+ "dateRange": { "date_from": "-1d" }+ }+}+```++Each row is one route. Compare `error_count / count` and `p95_duration_nano` across rows to find unhealthy endpoints.+## Aggregate within one service```jsondiff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex 2a5ac0bf9038..577eecf9d37e 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -1242,7 +1242,7 @@"feature_flag": "tracing"},"apm-spans-aggregate": {- "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.",+ "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n## query.groupByEndpoint\n\nSet to `true` to get per-endpoint health for HTTP servers. Rows then group by `METHOD route` (for example `GET /users/{id}`) from the `http.route` attribute, in place of the span name.\n\n- Only HTTP server spans (span kind server with an HTTP method) are counted. Queue consumers, workflow activities and client spans are left out.\n- A span without `http.route` keeps its span name as the row key.\n- Use it when span names are raw paths (`/users/1`, `/users/2`) or a bare method (`GET`), which split or merge routes.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Endpoint health for one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"groupByEndpoint\": true,\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nEach row is one route. Compare `error_count / count` and `p95_duration_nano` across rows to find unhealthy endpoints.\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.","category": "Tracing","feature": "tracing","summary": "Aggregate span statistics",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex 6ee512936214..9ffffc576288 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -1257,7 +1257,7 @@"feature_flag": "tracing"},"apm-spans-aggregate": {- "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.",+ "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n## query.groupByEndpoint\n\nSet to `true` to get per-endpoint health for HTTP servers. Rows then group by `METHOD route` (for example `GET /users/{id}`) from the `http.route` attribute, in place of the span name.\n\n- Only HTTP server spans (span kind server with an HTTP method) are counted. Queue consumers, workflow activities and client spans are left out.\n- A span without `http.route` keeps its span name as the row key.\n- Use it when span names are raw paths (`/users/1`, `/users/2`) or a bare method (`GET`), which split or merge routes.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Endpoint health for one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"groupByEndpoint\": true,\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nEach row is one route. Compare `error_count / count` and `p95_duration_nano` across rows to find unhealthy endpoints.\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.","category": "Tracing","feature": "tracing","summary": "Aggregate span statistics",diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex e9b540aaa761..bb3681f0e9dc 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -127306,6 +127306,8 @@ export namespace Schemas {offset?: number;/** Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches. */includeImpact?: boolean;+ /** Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `/users/1` and `/users/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out. */+ groupByEndpoint?: boolean;}export interface _TracingAggregationRequest {diff --git a/services/mcp/src/generated/tracing/api.ts b/services/mcp/src/generated/tracing/api.tsindex 73ba61a02e02..a332838fd08c 100644--- a/services/mcp/src/generated/tracing/api.ts+++ b/services/mcp/src/generated/tracing/api.ts@@ -23,6 +23,7 @@ export const tracingSpansAggregateCreateBodyQueryOneLimitMax = 5000export const tracingSpansAggregateCreateBodyQueryOneOffsetMin = 0export const tracingSpansAggregateCreateBodyQueryOneIncludeImpactDefault = false+export const tracingSpansAggregateCreateBodyQueryOneGroupByEndpointDefault = falseexport const TracingSpansAggregateCreateBody = () => zod.object({query: zod@@ -132,6 +133,12 @@ export const TracingSpansAggregateCreateBody = () => zod.object({.describe('Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches.'),+ groupByEndpoint: zod+ .boolean()+ .default(tracingSpansAggregateCreateBodyQueryOneGroupByEndpointDefault)+ .describe(+ 'Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `\/users\/1` and `\/users\/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out.'+ ),}).describe('The span aggregation query to execute.'),})diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/apm-spans-aggregate.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/apm-spans-aggregate.jsonindex 51988f17bd9a..cead78dd5995 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/apm-spans-aggregate.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/apm-spans-aggregate.json@@ -97,6 +97,11 @@},"type": "array"},+ "groupByEndpoint": {+ "default": false,+ "description": "Group HTTP server spans by `METHOD route` (from `http.route`) in place of the span name, so raw paths like `/users/1` and `/users/2` fold into one row. Spans without a route keep their span name. Spans that are not HTTP server spans are left out.",+ "type": "boolean"+ },"includeImpact": {"default": false,"description": "Also return the sessions and people behind each operation. Off by default because it reads the span and resource attribute maps, which the rest of the aggregation never touches.",