:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(tracing): match attribute filters only on spans that have the key (#124932)
Frank Hamand51fc05cc5d4a
diff --git a/products/tracing/backend/aggregation_query_runner.py b/products/tracing/backend/aggregation_query_runner.pyindex d917c3019019..3370429639cb 100644--- a/products/tracing/backend/aggregation_query_runner.py+++ b/products/tracing/backend/aggregation_query_runner.py@@ -55,7 +55,13 @@ from posthog.hogql_queries.utils.query_previous_period_date_range import QueryPr# _ROW_LIMIT lives in logic.py so the presentation layer can reach it through the# facade-allowed `logic` module; imported here (and re-exported) for the sibling runners.-from .logic import _ROW_LIMIT, TIME_BUCKET_DATE_RANGE_WHERE, translate_span_filter, with_span_attribute_type_suffix+from .logic import (+ _ROW_LIMIT,+ TIME_BUCKET_DATE_RANGE_WHERE,+ translate_span_filter,+ with_attribute_set_guards,+ with_span_attribute_type_suffix,+)from .models import TracingIdentityAttributeKeysfrom .span_identity import identity_value_expr@@ -226,8 +232,8 @@ class _SpanAggregationMixin:translate_span_filter(span_filter)exprs.append(property_to_expr(span_filter, team=self.team))if self.span_attribute_filters:- exprs.append(property_to_expr(self.span_attribute_filters, team=self.team))- for resource_filter in self.resource_attribute_filters:+ exprs.append(property_to_expr(with_attribute_set_guards(self.span_attribute_filters), team=self.team))+ for resource_filter in with_attribute_set_guards(self.resource_attribute_filters):exprs.append(property_to_expr(resource_filter, team=self.team))return ast.And(exprs=exprs)diff --git a/products/tracing/backend/logic.py b/products/tracing/backend/logic.pyindex f139fcba043f..de4d513bf8f5 100644--- a/products/tracing/backend/logic.py+++ b/products/tracing/backend/logic.py@@ -300,6 +300,28 @@ def parse_span_events(raw_events: list[str]) -> list[dict]:return [event for _, event in parsed]+_EXISTENCE_OPERATORS = frozenset({PropertyOperator.IS_SET, PropertyOperator.IS_NOT_SET})+++def with_attribute_set_guards(filters: list[SpanPropertyFilter]) -> list[SpanPropertyFilter]:+ """Return the attribute filters plus an is_set filter for each key that they compare.++ An attribute Map reads a missing key as its default value ('' or 0). Without the guard, a negated+ filter such as `not_icontains` matches every span that does not have the attribute. The trace list+ then selects a whole trace through one such span, so the trace shows even when its root span has a+ value that the filter excludes. The guard uses the same suffixed key as the filter, so it checks+ the Map that the comparison reads.+ """+ guards: dict[str, SpanPropertyFilter] = {}+ for prop in filters:+ # property_to_expr reads a filter without a value as "match all", so a guard would be its only effect.+ if prop.value is None or prop.value == []:+ continue+ if prop.operator not in _EXISTENCE_OPERATORS and prop.key not in guards:+ guards[prop.key] = SpanPropertyFilter(key=prop.key, type=prop.type, operator=PropertyOperator.IS_SET)+ return [*filters, *guards.values()]++class TraceSpansQueryRunnerMixin(QueryRunner):"""Shared WHERE clause and settings for all trace span query runners."""@@ -384,10 +406,10 @@ class TraceSpansQueryRunnerMixin(QueryRunner):exprs.append(property_to_expr(span_filter, team=self.team))if self.span_attribute_filters:- exprs.append(property_to_expr(self.span_attribute_filters, team=self.team))+ exprs.append(property_to_expr(with_attribute_set_guards(self.span_attribute_filters), team=self.team))if self.resource_attribute_filters:- for f in self.resource_attribute_filters:+ for f in with_attribute_set_guards(self.resource_attribute_filters):exprs.append(property_to_expr(f, team=self.team))if self.query.traceId:diff --git a/products/tracing/backend/tests/test_attribute_breakdown.py b/products/tracing/backend/tests/test_attribute_breakdown.pyindex 7d7b10ccdb9c..9c1813e1002d 100644--- a/products/tracing/backend/tests/test_attribute_breakdown.py+++ b/products/tracing/backend/tests/test_attribute_breakdown.py@@ -107,17 +107,29 @@ class TestTraceSpansAttributeBreakdown(_TraceSpansTestBase):by_value = {row["value"]: row["count"] for row in rows}self.assertEqual(by_value["api2.amplitude.com"], 5)- def test_filter_group_scopes_the_breakdown(self):- # The BubbleUp shape: breakdown of only the BAD spans.- rows = self._breakdown(- breakdownKey="server.address",- breakdownType="span_attribute",- filterGroup=[{"key": "status_code", "operator": "exact", "type": "span", "value": "Error"}],- )- self.assertEqual(- {(row["value"], row["count"]) for row in rows},- {("api2.amplitude.com", 4)},- )+ @parameterized.expand(+ [+ # The BubbleUp shape: breakdown of only the BAD spans.+ (+ "span_column",+ "server.address",+ "span_attribute",+ {"key": "status_code", "operator": "exact", "type": "span", "value": "Error"},+ {("api2.amplitude.com", 4)},+ ),+ # A negated attribute filter skips the pod-b span that has no server.address.+ (+ "negated_attribute_skips_spans_without_the_key",+ "k8s.pod.name",+ "span_resource_attribute",+ {"key": "server.address", "operator": "is_not", "type": "span_attribute", "value": "api.mixpanel.com"},+ {("pod-a", 5), ("pod-c", 1)},+ ),+ ]+ )+ def test_filter_group_scopes_the_breakdown(self, _name, breakdown_key, breakdown_type, prop, expected):+ rows = self._breakdown(breakdownKey=breakdown_key, breakdownType=breakdown_type, filterGroup=[prop])+ self.assertEqual({(row["value"], row["count"]) for row in rows}, expected)@parameterized.expand([diff --git a/products/tracing/backend/tests/test_span_attribute_filter.py b/products/tracing/backend/tests/test_span_attribute_filter.pyindex eff14b7a8f7a..f239864e2616 100644--- a/products/tracing/backend/tests/test_span_attribute_filter.py+++ b/products/tracing/backend/tests/test_span_attribute_filter.py@@ -102,9 +102,26 @@ class TestSpanAttributeFilter(ClickhouseTestMixin, APIBaseTest):},[WITH_CODE],),+ # A negated filter matches only spans that have the attribute. A span without the key would+ # otherwise read '' and select its whole trace in the trace list.("is_not",{"key": "code.filepath", "type": "span_attribute", "operator": "is_not", "value": "nope.rs"},+ [WITH_CODE],+ ),+ (+ "resource_is_not_on_missing_key",+ {+ "key": "deployment.environment",+ "type": "span_resource_attribute",+ "operator": "is_not",+ "value": "prod",+ },+ [],+ ),+ (+ "is_not_without_value_matches_all",+ {"key": "code.filepath", "type": "span_attribute", "operator": "is_not", "value": []},sorted([WITH_CODE, WITHOUT_CODE]),),("is_set", {"key": "code.filepath", "type": "span_attribute", "operator": "is_set"}, [WITH_CODE]),@@ -177,12 +194,12 @@ class TestSpanAttributeFilter(ClickhouseTestMixin, APIBaseTest):{"key": "code.lineno", "type": "span_attribute", "operator": "between", "value": ["50", "60"]},[],),- # not_between keeps spans missing the attribute: a row without code.lineno is not inside- # [0, 41], so negating the range must return it alongside the out-of-range WITH_CODE span.+ # not_between matches only spans that have the attribute, so the span without code.lineno+ # is not returned with the out-of-range WITH_CODE span.("lineno_not_between_matches",{"key": "code.lineno", "type": "span_attribute", "operator": "not_between", "value": ["0", "41"]},- sorted([WITH_CODE, WITHOUT_CODE]),+ [WITH_CODE],),])diff --git a/products/tracing/mcp/prompts/apm-spans-aggregate.md b/products/tracing/mcp/prompts/apm-spans-aggregate.mdindex 6bc0b8947140..ba00dc5ccf7a 100644--- a/products/tracing/mcp/prompts/apm-spans-aggregate.md+++ b/products/tracing/mcp/prompts/apm-spans-aggregate.md@@ -52,6 +52,8 @@ Supported operators:- Numeric: `exact`, `gt`, `lt`- Existence (no value needed): `is_set`, `is_not_set`+An attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.+## Time periodUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.diff --git a/products/tracing/mcp/prompts/query-apm-spans.md b/products/tracing/mcp/prompts/query-apm-spans.mdindex b68aa01315f2..1952d56c86cb 100644--- a/products/tracing/mcp/prompts/query-apm-spans.md+++ b/products/tracing/mcp/prompts/query-apm-spans.md@@ -43,6 +43,8 @@ Supported operators:The `value` field accepts a string, number, or array of strings depending on the operator. Omit `value` for `is_set`/`is_not_set`.+An attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.+## Time periodUse the `query.dateRange` field to control the time window. If the question doesn't mention time, the default is the last hour (`-1h`). Examples of relative dates: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex a0e5d13fe25a..c04004e6e060 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -1242,7 +1242,7 @@"feature_flag": "tracing"},"apm-spans-aggregate": {- "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.",+ "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.","category": "Tracing","feature": "tracing","summary": "Aggregate span statistics",@@ -11693,7 +11693,7 @@"feature_flag": "engineering-analytics"},"query-apm-spans": {- "description": "Query trace spans with filtering by service name, status code, date range, and structured attribute filters. Supports cursor-based pagination. Returns spans with uuid, trace_id, span_id, parent_span_id, name, kind, service_name, status_code, timestamp, end_time, duration_nano, is_root_span, matched_filter, and attributes (the span-level OTel attribute map, e.g. db.statement, http.url).\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nUse 'apm-attributes-list' and 'apm-attribute-values-list' to discover available attributes before building filters. Use 'apm-services-list' to discover available services.\n\n# Return shape\n\nResults are **grouped by trace**, not a flat list of matching spans. For each trace that contains at least one span matching your filters, the response includes spans from that trace (up to `prefetchSpans` per trace, root span first). Two fields tell you which spans actually matched:\n\n- `matched_filter` — `1` if **this span** satisfies your `filterGroup`/`serviceNames`/`statusCodes`, `0` if it's only included because it shares a trace with a match (e.g. a prefetched sibling or the trace's root). When you filter by a child span's name, the matching child has `matched_filter: 1` and its root/siblings have `matched_filter: 0`.\n- `is_root_span` — `true` for the trace's entry span.\n\nTo collapse each matching trace to a **single row — its root span**, set `rootSpans: true` — see below. (The row is the trace's entry span, which may itself carry `matched_filter: 0` when the match was on a child.) To inspect a single trace's full tree, take a `trace_id` from the results and call `apm-trace-get`.\n\nCRITICAL: Be minimalist. Only include filters and settings that are essential to answer the user's specific question. Default settings are usually sufficient unless the user explicitly requests customization.\n\n# Data narrowing\n\n## Property filters\n\nUse property filters via the `query.filterGroup` field to narrow results. Only include property filters when they are essential to directly answer the user's question.\n\nWhen using a property filter, you should:\n\n- **Choose the right type.** Span property types are:\n - `span` — filters built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n - `span_attribute` — filters span-level attributes (e.g. \"http.method\", \"http.status_code\").\n - `span_resource_attribute` — filters resource-level attributes (e.g. k8s labels, deployment info).\n- **Use `apm-attributes-list` to discover available attribute keys** before building filters.\n- **Use `apm-attribute-values-list` to discover valid values** for a specific attribute key.\n- **Find the suitable operator for the value type** (see supported operators below).\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nThe `value` field accepts a string, number, or array of strings depending on the operator. Omit `value` for `is_set`/`is_not_set`.\n\n## Time period\n\nUse the `query.dateRange` field to control the time window. If the question doesn't mention time, the default is the last hour (`-1h`). Examples of relative dates: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n\n# Parameters\n\n## query.serviceNames\n\nFilter by service names. Use `apm-services-list` to discover available services.\n\n## query.statusCodes\n\nFilter by OTel span status codes (list of integers: `0` Unset, `1` OK, `2` Error) — **not** HTTP status codes. Use `[2]` to select error spans.\n\n## query.orderBy\n\nSort by timestamp: `latest` (default) or `earliest`.\n\n## query.filterGroup\n\nA list of property filters to narrow results. Each filter specifies `key`, `operator`, `type` (span/span_attribute/span_resource_attribute), and optionally `value`. See the \"Property filters\" section above.\n\n## query.dateRange\n\nDate range to filter results. Defaults to the last hour (`-1h`).\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.traceId\n\nFilter to a specific trace ID (hex string). Use this when you already know the trace ID.\n\n## query.rootSpans\n\nSet `true` to return **only root spans** — one entry span per matching trace, which collapses each trace to a single row. Useful for \"list the traces matching X\" without sifting through `matched_filter`. Leave unset (or `false`) to get all spans of matching traces, where you read `matched_filter` to find the ones that matched. The frontend leaves this unset.\n\n## query.flatSpans\n\nSet `true` to return **the matching spans themselves, one row per span** (root and child), rather than collapsing to traces. This is the way to search by a child-span attribute (e.g. `code.filepath`) — the result is the matching child spans directly, not the traces that contain them. Streams under `ORDER BY … LIMIT`, so it stays bounded on hot child attributes where the whole-trace grouping would not. Distinct from `rootSpans` (which scopes whole-trace selection); `prefetchSpans` is ignored. Defaults to false.\n\n## query.limit\n\nMaximum number of results (1-1000). Defaults to 100.\n\n## query.after\n\nCursor for pagination. Use the `nextCursor` value from the previous response.\n\n## query.prefetchSpans\n\nNumber of spans to return per matching trace (1-100), root span first. Useful to preview trace structure without a separate `apm-trace-get`. With the default (1) you get one span per trace (the root); raise it to also pull the matching children and their siblings (check `matched_filter` to tell them apart). Ignored when `rootSpans: true`, which always returns just the root.\n\n## query.excludeAttributes\n\nSet `true` to drop the per-span `attributes` map from results (the map stays present but empty). The attribute map holds multi-KB values like `db.statement`, so excluding it keeps large result sets compact — set it when you only need span structure/timing (`name`, `service_name`, `duration_nano`, `parent_span_id`) and not the OTel attributes. Defaults to false.\n\n# Examples\n\n## List recent error spans\n\n```json\n{\n \"query\": {\n \"statusCodes\": [2]\n }\n}\n```\n\n## Search spans from a specific service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n## Filter by a span attribute\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"http.method\", \"operator\": \"exact\", \"type\": \"span_attribute\", \"value\": \"POST\" }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Find slow spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"duration\", \"operator\": \"gt\", \"type\": \"span\", \"value\": \"1000\" }],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n## Combine service and attribute filters\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"web-server\"],\n \"filterGroup\": [{ \"key\": \"http.status_code\", \"operator\": \"gt\", \"type\": \"span_attribute\", \"value\": \"399\" }],\n \"dateRange\": { \"date_from\": \"-12h\" }\n }\n}\n```\n\n## Check if a resource attribute exists\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"k8s.pod.name\", \"operator\": \"is_set\", \"type\": \"span_resource_attribute\" }]\n }\n}\n```\n\n# Reminders\n\n- Ensure that any property filters are directly relevant to the user's question. Avoid unnecessary filtering.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attributes before guessing filter keys/values.\n- Use `apm-services-list` to discover available services before filtering by service name.\n- A `duration` filter value is in milliseconds (1 second = `1000`). The `duration_nano` response field is in nanoseconds (1 second = 1,000,000,000).",+ "description": "Query trace spans with filtering by service name, status code, date range, and structured attribute filters. Supports cursor-based pagination. Returns spans with uuid, trace_id, span_id, parent_span_id, name, kind, service_name, status_code, timestamp, end_time, duration_nano, is_root_span, matched_filter, and attributes (the span-level OTel attribute map, e.g. db.statement, http.url).\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nUse 'apm-attributes-list' and 'apm-attribute-values-list' to discover available attributes before building filters. Use 'apm-services-list' to discover available services.\n\n# Return shape\n\nResults are **grouped by trace**, not a flat list of matching spans. For each trace that contains at least one span matching your filters, the response includes spans from that trace (up to `prefetchSpans` per trace, root span first). Two fields tell you which spans actually matched:\n\n- `matched_filter` — `1` if **this span** satisfies your `filterGroup`/`serviceNames`/`statusCodes`, `0` if it's only included because it shares a trace with a match (e.g. a prefetched sibling or the trace's root). When you filter by a child span's name, the matching child has `matched_filter: 1` and its root/siblings have `matched_filter: 0`.\n- `is_root_span` — `true` for the trace's entry span.\n\nTo collapse each matching trace to a **single row — its root span**, set `rootSpans: true` — see below. (The row is the trace's entry span, which may itself carry `matched_filter: 0` when the match was on a child.) To inspect a single trace's full tree, take a `trace_id` from the results and call `apm-trace-get`.\n\nCRITICAL: Be minimalist. Only include filters and settings that are essential to answer the user's specific question. Default settings are usually sufficient unless the user explicitly requests customization.\n\n# Data narrowing\n\n## Property filters\n\nUse property filters via the `query.filterGroup` field to narrow results. Only include property filters when they are essential to directly answer the user's question.\n\nWhen using a property filter, you should:\n\n- **Choose the right type.** Span property types are:\n - `span` — filters built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n - `span_attribute` — filters span-level attributes (e.g. \"http.method\", \"http.status_code\").\n - `span_resource_attribute` — filters resource-level attributes (e.g. k8s labels, deployment info).\n- **Use `apm-attributes-list` to discover available attribute keys** before building filters.\n- **Use `apm-attribute-values-list` to discover valid values** for a specific attribute key.\n- **Find the suitable operator for the value type** (see supported operators below).\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nThe `value` field accepts a string, number, or array of strings depending on the operator. Omit `value` for `is_set`/`is_not_set`.\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse the `query.dateRange` field to control the time window. If the question doesn't mention time, the default is the last hour (`-1h`). Examples of relative dates: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n\n# Parameters\n\n## query.serviceNames\n\nFilter by service names. Use `apm-services-list` to discover available services.\n\n## query.statusCodes\n\nFilter by OTel span status codes (list of integers: `0` Unset, `1` OK, `2` Error) — **not** HTTP status codes. Use `[2]` to select error spans.\n\n## query.orderBy\n\nSort by timestamp: `latest` (default) or `earliest`.\n\n## query.filterGroup\n\nA list of property filters to narrow results. Each filter specifies `key`, `operator`, `type` (span/span_attribute/span_resource_attribute), and optionally `value`. See the \"Property filters\" section above.\n\n## query.dateRange\n\nDate range to filter results. Defaults to the last hour (`-1h`).\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.traceId\n\nFilter to a specific trace ID (hex string). Use this when you already know the trace ID.\n\n## query.rootSpans\n\nSet `true` to return **only root spans** — one entry span per matching trace, which collapses each trace to a single row. Useful for \"list the traces matching X\" without sifting through `matched_filter`. Leave unset (or `false`) to get all spans of matching traces, where you read `matched_filter` to find the ones that matched. The frontend leaves this unset.\n\n## query.flatSpans\n\nSet `true` to return **the matching spans themselves, one row per span** (root and child), rather than collapsing to traces. This is the way to search by a child-span attribute (e.g. `code.filepath`) — the result is the matching child spans directly, not the traces that contain them. Streams under `ORDER BY … LIMIT`, so it stays bounded on hot child attributes where the whole-trace grouping would not. Distinct from `rootSpans` (which scopes whole-trace selection); `prefetchSpans` is ignored. Defaults to false.\n\n## query.limit\n\nMaximum number of results (1-1000). Defaults to 100.\n\n## query.after\n\nCursor for pagination. Use the `nextCursor` value from the previous response.\n\n## query.prefetchSpans\n\nNumber of spans to return per matching trace (1-100), root span first. Useful to preview trace structure without a separate `apm-trace-get`. With the default (1) you get one span per trace (the root); raise it to also pull the matching children and their siblings (check `matched_filter` to tell them apart). Ignored when `rootSpans: true`, which always returns just the root.\n\n## query.excludeAttributes\n\nSet `true` to drop the per-span `attributes` map from results (the map stays present but empty). The attribute map holds multi-KB values like `db.statement`, so excluding it keeps large result sets compact — set it when you only need span structure/timing (`name`, `service_name`, `duration_nano`, `parent_span_id`) and not the OTel attributes. Defaults to false.\n\n# Examples\n\n## List recent error spans\n\n```json\n{\n \"query\": {\n \"statusCodes\": [2]\n }\n}\n```\n\n## Search spans from a specific service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n## Filter by a span attribute\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"http.method\", \"operator\": \"exact\", \"type\": \"span_attribute\", \"value\": \"POST\" }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Find slow spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"duration\", \"operator\": \"gt\", \"type\": \"span\", \"value\": \"1000\" }],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n## Combine service and attribute filters\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"web-server\"],\n \"filterGroup\": [{ \"key\": \"http.status_code\", \"operator\": \"gt\", \"type\": \"span_attribute\", \"value\": \"399\" }],\n \"dateRange\": { \"date_from\": \"-12h\" }\n }\n}\n```\n\n## Check if a resource attribute exists\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"k8s.pod.name\", \"operator\": \"is_set\", \"type\": \"span_resource_attribute\" }]\n }\n}\n```\n\n# Reminders\n\n- Ensure that any property filters are directly relevant to the user's question. Avoid unnecessary filtering.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attributes before guessing filter keys/values.\n- Use `apm-services-list` to discover available services before filtering by service name.\n- A `duration` filter value is in milliseconds (1 second = `1000`). The `duration_nano` response field is in nanoseconds (1 second = 1,000,000,000).","category": "Tracing","feature": "tracing","summary": "Query tracing spans",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex ccac065db398..a2198eefdbfd 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -1257,7 +1257,7 @@"feature_flag": "tracing"},"apm-spans-aggregate": {- "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.",+ "description": "Aggregate trace span statistics grouped by `(service_name, name)` over a date window.\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nReturns one row per `(service_name, name)` pair with the following metrics:\n\n- `count` — number of spans matched\n- `total_duration_nano`, `avg_duration_nano`, `p50_duration_nano`, `p95_duration_nano`, `p99_duration_nano`, `p999_duration_nano` — duration stats in nanoseconds (1 second = 1,000,000,000 ns). `p999_duration_nano` is the 99.9th percentile and is only meaningful for `(service, name)` groups with enough spans; on low-volume operations it collapses to the max\n- `error_count` — spans with OTel status code `Error` (status_code = 2)\n\nRows are ordered by `total_duration_nano` DESC. By default only the top 100 rows are returned — this keeps the response small, which matters because high span-name cardinality (e.g. untemplated URL paths) can otherwise produce an enormous payload. The top rows already surface the heaviest operations. When more rows genuinely exist the response sets `has_more: true` and `next_offset`; page through with `offset`, or raise `limit` (hard max 5000). Prefer narrowing with `serviceNames`/`filterGroup` over pulling a large page.\n\nUse to answer:\n\n- \"Which services/operations consume the most time?\"\n- \"What's the p95 or p99 latency of `GET /users`?\"\n- \"How many errors did the checkout service emit in the last day?\"\n- \"Did `POST /orders` get slower this week vs last week?\" (with `compareFilter`)\n\nFor per-call-tree breakdowns (parent → child relationships), use `apm-spans-tree` instead. For time-bucketed trends (\"when did it change?\"), use `apm-spans-sparkline` instead — `compareFilter` only contrasts two static windows.\n\n# Comparison window\n\nSet `query.compareFilter.compare: true` to also fetch a comparison window. The response then includes a `compare` array of the same shape as `results`.\n\n- Omit `compare_to` (or set null) to compare against the immediately previous period of equal length (e.g. `dateRange: -1d` → compares vs the day before).\n- Set `compare_to: \"-7d\"` to compare against the window 7 days earlier (same length as the primary window).\n\nWhen `compare` is true, `compare` is always an array — empty (`[]`) if the comparison window matched no spans (for example an operation that didn't exist yet), not null. `compare` is `null` only when no comparison was requested. So an empty array is a real answer (\"nothing in the prior window\"), not a failure.\n\n# Data narrowing\n\n## Property filters\n\nUse `query.filterGroup` to narrow results to spans matching specific attributes. Only include filters that are essential to the user's question.\n\nFilter `type` values:\n\n- `span` — built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n- `span_attribute` — span-level attributes (e.g. \"http.method\", \"http.status_code\")\n- `span_resource_attribute` — resource-level attributes (e.g. k8s labels, deployment info)\n\nUse `apm-attributes-list` and `apm-attribute-values-list` to discover available attribute keys/values before guessing.\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse `query.dateRange` to control the time window. Default is the last hour (`-1h`). Examples: `-1h`, `-6h`, `-1d`, `-7d`.\n\n# Parameters\n\n## query.dateRange\n\nDate range for the primary window. Defaults to the last hour.\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.compareFilter\n\nOptional comparison-window configuration. Omit when you only need the primary window.\n\n- `compare` (boolean): set to true to enable comparison.\n- `compare_to` (string, optional): relative offset for the comparison window (e.g. `-1d`, `-7d`). Defaults to the immediately previous period of equal length.\n\n## query.serviceNames\n\nList of service names to restrict the aggregation to. Use `apm-services-list` to discover available services.\n\n## query.filterGroup\n\nA flat list of property filters applied to both the primary and comparison windows. See the \"Property filters\" section.\n\n## query.limit\n\nMax rows to return, ordered by `total_duration_nano` DESC. Defaults to 100; hard max 5000. Keep this small — a high value on high-cardinality span names returns a very large response. Narrow with `serviceNames`/`filterGroup` instead of raising it when you can.\n\n## query.offset\n\nRow offset for pagination. Combine with `limit` and the `next_offset` from a previous response to page through results beyond the first page.\n\n# Examples\n\n## Top operations by total time in the last hour\n\n```json\n{\n \"query\": {}\n}\n```\n\n## p95 latency by operation in the last day\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\nInspect the `p95_duration_nano` field on each result.\n\n## Compare this hour vs the previous hour\n\n```json\n{\n \"query\": {\n \"compareFilter\": { \"compare\": true }\n }\n}\n```\n\nReturns `results` for the last hour and `compare` for the hour before. Diff `total_duration_nano`, `avg_duration_nano`, etc. to find regressions.\n\n## Compare today vs same time last week\n\n```json\n{\n \"query\": {\n \"dateRange\": { \"date_from\": \"-1d\" },\n \"compareFilter\": { \"compare\": true, \"compare_to\": \"-7d\" }\n }\n}\n```\n\n## Aggregate only error spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"status_code\", \"operator\": \"exact\", \"type\": \"span\", \"value\": 2 }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Aggregate within one service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n# Reminders\n\n- Duration values in results are in nanoseconds. Divide by 1,000,000 for ms, 1,000,000,000 for seconds. A `duration` filter value is in milliseconds.\n- Results are ordered by `total_duration_nano` DESC and default to the top 100 rows. Check `has_more`/`next_offset` to page; raise `limit` (max 5000) only when you truly need the long tail.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attribute keys/values before filtering.\n- Use `apm-services-list` to discover services before filtering by service name.\n- For parent → child breakdowns, use `apm-spans-tree` instead.","category": "Tracing","feature": "tracing","summary": "Aggregate span statistics",@@ -12380,7 +12380,7 @@"feature_flag": "engineering-analytics"},"query-apm-spans": {- "description": "Query trace spans with filtering by service name, status code, date range, and structured attribute filters. Supports cursor-based pagination. Returns spans with uuid, trace_id, span_id, parent_span_id, name, kind, service_name, status_code, timestamp, end_time, duration_nano, is_root_span, matched_filter, and attributes (the span-level OTel attribute map, e.g. db.statement, http.url).\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nUse 'apm-attributes-list' and 'apm-attribute-values-list' to discover available attributes before building filters. Use 'apm-services-list' to discover available services.\n\n# Return shape\n\nResults are **grouped by trace**, not a flat list of matching spans. For each trace that contains at least one span matching your filters, the response includes spans from that trace (up to `prefetchSpans` per trace, root span first). Two fields tell you which spans actually matched:\n\n- `matched_filter` — `1` if **this span** satisfies your `filterGroup`/`serviceNames`/`statusCodes`, `0` if it's only included because it shares a trace with a match (e.g. a prefetched sibling or the trace's root). When you filter by a child span's name, the matching child has `matched_filter: 1` and its root/siblings have `matched_filter: 0`.\n- `is_root_span` — `true` for the trace's entry span.\n\nTo collapse each matching trace to a **single row — its root span**, set `rootSpans: true` — see below. (The row is the trace's entry span, which may itself carry `matched_filter: 0` when the match was on a child.) To inspect a single trace's full tree, take a `trace_id` from the results and call `apm-trace-get`.\n\nCRITICAL: Be minimalist. Only include filters and settings that are essential to answer the user's specific question. Default settings are usually sufficient unless the user explicitly requests customization.\n\n# Data narrowing\n\n## Property filters\n\nUse property filters via the `query.filterGroup` field to narrow results. Only include property filters when they are essential to directly answer the user's question.\n\nWhen using a property filter, you should:\n\n- **Choose the right type.** Span property types are:\n - `span` — filters built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n - `span_attribute` — filters span-level attributes (e.g. \"http.method\", \"http.status_code\").\n - `span_resource_attribute` — filters resource-level attributes (e.g. k8s labels, deployment info).\n- **Use `apm-attributes-list` to discover available attribute keys** before building filters.\n- **Use `apm-attribute-values-list` to discover valid values** for a specific attribute key.\n- **Find the suitable operator for the value type** (see supported operators below).\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nThe `value` field accepts a string, number, or array of strings depending on the operator. Omit `value` for `is_set`/`is_not_set`.\n\n## Time period\n\nUse the `query.dateRange` field to control the time window. If the question doesn't mention time, the default is the last hour (`-1h`). Examples of relative dates: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n\n# Parameters\n\n## query.serviceNames\n\nFilter by service names. Use `apm-services-list` to discover available services.\n\n## query.statusCodes\n\nFilter by OTel span status codes (list of integers: `0` Unset, `1` OK, `2` Error) — **not** HTTP status codes. Use `[2]` to select error spans.\n\n## query.orderBy\n\nSort by timestamp: `latest` (default) or `earliest`.\n\n## query.filterGroup\n\nA list of property filters to narrow results. Each filter specifies `key`, `operator`, `type` (span/span_attribute/span_resource_attribute), and optionally `value`. See the \"Property filters\" section above.\n\n## query.dateRange\n\nDate range to filter results. Defaults to the last hour (`-1h`).\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.traceId\n\nFilter to a specific trace ID (hex string). Use this when you already know the trace ID.\n\n## query.rootSpans\n\nSet `true` to return **only root spans** — one entry span per matching trace, which collapses each trace to a single row. Useful for \"list the traces matching X\" without sifting through `matched_filter`. Leave unset (or `false`) to get all spans of matching traces, where you read `matched_filter` to find the ones that matched. The frontend leaves this unset.\n\n## query.flatSpans\n\nSet `true` to return **the matching spans themselves, one row per span** (root and child), rather than collapsing to traces. This is the way to search by a child-span attribute (e.g. `code.filepath`) — the result is the matching child spans directly, not the traces that contain them. Streams under `ORDER BY … LIMIT`, so it stays bounded on hot child attributes where the whole-trace grouping would not. Distinct from `rootSpans` (which scopes whole-trace selection); `prefetchSpans` is ignored. Defaults to false.\n\n## query.limit\n\nMaximum number of results (1-1000). Defaults to 100.\n\n## query.after\n\nCursor for pagination. Use the `nextCursor` value from the previous response.\n\n## query.prefetchSpans\n\nNumber of spans to return per matching trace (1-100), root span first. Useful to preview trace structure without a separate `apm-trace-get`. With the default (1) you get one span per trace (the root); raise it to also pull the matching children and their siblings (check `matched_filter` to tell them apart). Ignored when `rootSpans: true`, which always returns just the root.\n\n## query.excludeAttributes\n\nSet `true` to drop the per-span `attributes` map from results (the map stays present but empty). The attribute map holds multi-KB values like `db.statement`, so excluding it keeps large result sets compact — set it when you only need span structure/timing (`name`, `service_name`, `duration_nano`, `parent_span_id`) and not the OTel attributes. Defaults to false.\n\n# Examples\n\n## List recent error spans\n\n```json\n{\n \"query\": {\n \"statusCodes\": [2]\n }\n}\n```\n\n## Search spans from a specific service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n## Filter by a span attribute\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"http.method\", \"operator\": \"exact\", \"type\": \"span_attribute\", \"value\": \"POST\" }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Find slow spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"duration\", \"operator\": \"gt\", \"type\": \"span\", \"value\": \"1000\" }],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n## Combine service and attribute filters\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"web-server\"],\n \"filterGroup\": [{ \"key\": \"http.status_code\", \"operator\": \"gt\", \"type\": \"span_attribute\", \"value\": \"399\" }],\n \"dateRange\": { \"date_from\": \"-12h\" }\n }\n}\n```\n\n## Check if a resource attribute exists\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"k8s.pod.name\", \"operator\": \"is_set\", \"type\": \"span_resource_attribute\" }]\n }\n}\n```\n\n# Reminders\n\n- Ensure that any property filters are directly relevant to the user's question. Avoid unnecessary filtering.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attributes before guessing filter keys/values.\n- Use `apm-services-list` to discover available services before filtering by service name.\n- A `duration` filter value is in milliseconds (1 second = `1000`). The `duration_nano` response field is in nanoseconds (1 second = 1,000,000,000).",+ "description": "Query trace spans with filtering by service name, status code, date range, and structured attribute filters. Supports cursor-based pagination. Returns spans with uuid, trace_id, span_id, parent_span_id, name, kind, service_name, status_code, timestamp, end_time, duration_nano, is_root_span, matched_filter, and attributes (the span-level OTel attribute map, e.g. db.statement, http.url).\n\nAll parameters go inside `query` — top-level fields are rejected:\n\n```json\n{ \"query\": { \"serviceNames\": [\"api\"], \"dateRange\": { \"date_from\": \"-1h\" } } }\n```\n\nUse 'apm-attributes-list' and 'apm-attribute-values-list' to discover available attributes before building filters. Use 'apm-services-list' to discover available services.\n\n# Return shape\n\nResults are **grouped by trace**, not a flat list of matching spans. For each trace that contains at least one span matching your filters, the response includes spans from that trace (up to `prefetchSpans` per trace, root span first). Two fields tell you which spans actually matched:\n\n- `matched_filter` — `1` if **this span** satisfies your `filterGroup`/`serviceNames`/`statusCodes`, `0` if it's only included because it shares a trace with a match (e.g. a prefetched sibling or the trace's root). When you filter by a child span's name, the matching child has `matched_filter: 1` and its root/siblings have `matched_filter: 0`.\n- `is_root_span` — `true` for the trace's entry span.\n\nTo collapse each matching trace to a **single row — its root span**, set `rootSpans: true` — see below. (The row is the trace's entry span, which may itself carry `matched_filter: 0` when the match was on a child.) To inspect a single trace's full tree, take a `trace_id` from the results and call `apm-trace-get`.\n\nCRITICAL: Be minimalist. Only include filters and settings that are essential to answer the user's specific question. Default settings are usually sufficient unless the user explicitly requests customization.\n\n# Data narrowing\n\n## Property filters\n\nUse property filters via the `query.filterGroup` field to narrow results. Only include property filters when they are essential to directly answer the user's question.\n\nWhen using a property filter, you should:\n\n- **Choose the right type.** Span property types are:\n - `span` — filters built-in span fields (trace_id, span_id, duration, name, kind, status_code, is_root_span). A `duration` filter value is in milliseconds (1 second = `1000`).\n - `span_attribute` — filters span-level attributes (e.g. \"http.method\", \"http.status_code\").\n - `span_resource_attribute` — filters resource-level attributes (e.g. k8s labels, deployment info).\n- **Use `apm-attributes-list` to discover available attribute keys** before building filters.\n- **Use `apm-attribute-values-list` to discover valid values** for a specific attribute key.\n- **Find the suitable operator for the value type** (see supported operators below).\n\nSupported operators:\n\n- String: `exact`, `is_not`, `icontains`, `not_icontains`, `regex`, `not_regex`\n- Numeric: `exact`, `gt`, `lt`\n- Existence (no value needed): `is_set`, `is_not_set`\n\nThe `value` field accepts a string, number, or array of strings depending on the operator. Omit `value` for `is_set`/`is_not_set`.\n\nAn attribute filter (`span_attribute` or `span_resource_attribute`) matches only spans that have the attribute, also for negated operators such as `is_not` and `not_icontains`. To find spans without the attribute, use `is_not_set`.\n\n## Time period\n\nUse the `query.dateRange` field to control the time window. If the question doesn't mention time, the default is the last hour (`-1h`). Examples of relative dates: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n\n# Parameters\n\n## query.serviceNames\n\nFilter by service names. Use `apm-services-list` to discover available services.\n\n## query.statusCodes\n\nFilter by OTel span status codes (list of integers: `0` Unset, `1` OK, `2` Error) — **not** HTTP status codes. Use `[2]` to select error spans.\n\n## query.orderBy\n\nSort by timestamp: `latest` (default) or `earliest`.\n\n## query.filterGroup\n\nA list of property filters to narrow results. Each filter specifies `key`, `operator`, `type` (span/span_attribute/span_resource_attribute), and optionally `value`. See the \"Property filters\" section above.\n\n## query.dateRange\n\nDate range to filter results. Defaults to the last hour (`-1h`).\n\n- `date_from`: Start of the range. Accepts ISO 8601 timestamps or relative formats: `-1h`, `-6h`, `-1d`, `-7d`, `-30d`.\n- `date_to`: End of the range. Same format. Omit or null for \"now\".\n\n## query.traceId\n\nFilter to a specific trace ID (hex string). Use this when you already know the trace ID.\n\n## query.rootSpans\n\nSet `true` to return **only root spans** — one entry span per matching trace, which collapses each trace to a single row. Useful for \"list the traces matching X\" without sifting through `matched_filter`. Leave unset (or `false`) to get all spans of matching traces, where you read `matched_filter` to find the ones that matched. The frontend leaves this unset.\n\n## query.flatSpans\n\nSet `true` to return **the matching spans themselves, one row per span** (root and child), rather than collapsing to traces. This is the way to search by a child-span attribute (e.g. `code.filepath`) — the result is the matching child spans directly, not the traces that contain them. Streams under `ORDER BY … LIMIT`, so it stays bounded on hot child attributes where the whole-trace grouping would not. Distinct from `rootSpans` (which scopes whole-trace selection); `prefetchSpans` is ignored. Defaults to false.\n\n## query.limit\n\nMaximum number of results (1-1000). Defaults to 100.\n\n## query.after\n\nCursor for pagination. Use the `nextCursor` value from the previous response.\n\n## query.prefetchSpans\n\nNumber of spans to return per matching trace (1-100), root span first. Useful to preview trace structure without a separate `apm-trace-get`. With the default (1) you get one span per trace (the root); raise it to also pull the matching children and their siblings (check `matched_filter` to tell them apart). Ignored when `rootSpans: true`, which always returns just the root.\n\n## query.excludeAttributes\n\nSet `true` to drop the per-span `attributes` map from results (the map stays present but empty). The attribute map holds multi-KB values like `db.statement`, so excluding it keeps large result sets compact — set it when you only need span structure/timing (`name`, `service_name`, `duration_nano`, `parent_span_id`) and not the OTel attributes. Defaults to false.\n\n# Examples\n\n## List recent error spans\n\n```json\n{\n \"query\": {\n \"statusCodes\": [2]\n }\n}\n```\n\n## Search spans from a specific service\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"api-gateway\"],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n## Filter by a span attribute\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"http.method\", \"operator\": \"exact\", \"type\": \"span_attribute\", \"value\": \"POST\" }],\n \"dateRange\": { \"date_from\": \"-6h\" }\n }\n}\n```\n\n## Find slow spans\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"duration\", \"operator\": \"gt\", \"type\": \"span\", \"value\": \"1000\" }],\n \"dateRange\": { \"date_from\": \"-1d\" }\n }\n}\n```\n\n## Combine service and attribute filters\n\n```json\n{\n \"query\": {\n \"serviceNames\": [\"web-server\"],\n \"filterGroup\": [{ \"key\": \"http.status_code\", \"operator\": \"gt\", \"type\": \"span_attribute\", \"value\": \"399\" }],\n \"dateRange\": { \"date_from\": \"-12h\" }\n }\n}\n```\n\n## Check if a resource attribute exists\n\n```json\n{\n \"query\": {\n \"filterGroup\": [{ \"key\": \"k8s.pod.name\", \"operator\": \"is_set\", \"type\": \"span_resource_attribute\" }]\n }\n}\n```\n\n# Reminders\n\n- Ensure that any property filters are directly relevant to the user's question. Avoid unnecessary filtering.\n- Use `apm-attributes-list` and `apm-attribute-values-list` to discover attributes before guessing filter keys/values.\n- Use `apm-services-list` to discover available services before filtering by service name.\n- A `duration` filter value is in milliseconds (1 second = `1000`). The `duration_nano` response field is in nanoseconds (1 second = 1,000,000,000).","category": "Tracing","feature": "tracing","summary": "Query tracing spans",