:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(tracing): show multi-root traces as one row with full duration (#128139)
Frank Hamand552bf52c7852
diff --git a/products/tracing/backend/logic.py b/products/tracing/backend/logic.pyindex 8ed44dbdd6b5..fb7558af787c 100644--- a/products/tracing/backend/logic.py+++ b/products/tracing/backend/logic.py@@ -63,6 +63,8 @@ TIME_BUCKET_DATE_RANGE_WHERE = (# before the range, so the trace does not show as having no root.ROOT_SPAN_LOOKBACK = dt.timedelta(minutes=15)+TRACE_DURATION_NANO_SQL = "dateDiff('microsecond', min(timestamp), max(end_time)) * 1000"+# Hard cap on number of rows returned per period by the span aggregation runners. Keeps# payloads bounded when name cardinality blows up (e.g. untemplated URL paths). The flame# graph collapses long tails anyway so the lower-ranked rows aren't visible.@@ -668,9 +670,9 @@ class TraceSpansQueryRunner(TraceSpansQueryRunnerMixin, AnalyticsQueryRunner[Tra# per-trace value — `LIMIT 1 BY trace_id` can't paginate cleanly because a multi-span trace# straddling the boundary would be re-selected via its other spans. Time sorts order by each# trace's start time (`min(timestamp)`) and keyset on it; duration sorts order by trace- # duration (`max(duration_nano)`) and offset-paginate (the time index can't prune a duration- # order, so keyset would pay its cost for none of its benefit).- sort_key_sql = "max(duration_nano)" if by_duration else "min(timestamp)"+ # duration (first span start to last span end) and offset-paginate (the time index can't prune+ # a duration order, so keyset would pay its cost for none of its benefit).+ sort_key_sql = TRACE_DURATION_NANO_SQL if by_duration else "min(timestamp)"# rootSpans is gated on `is True` (not truthiness), so None leaves the prefetch-driven# waterfall untouched. The list endpoint defaults it to True, and the span list sends False@@ -757,7 +759,7 @@ class TraceSpansQueryRunner(TraceSpansQueryRunnerMixin, AnalyticsQueryRunner[Tra# `trace_start` / `trace_duration` are the per-trace keys the view paginates and re-sorts on.# They MUST aggregate over the same rows the trace-selection subquery grouped, or the keys# diverge from the set the subquery picked — e.g. under root_only the subquery orders traces- # by `max(duration_nano)` over root spans, so a window over *all* spans would rank a trace by+ # by the duration of their root spans, so a window over *all* spans would rank a trace by# a long child the subquery never considered, corrupting the order and offset page boundaries.# Scope the key windows to match the subquery (cursor/time bounds are deliberately excluded —# the key is a property of the whole trace, not of the current page).@@ -817,7 +819,11 @@ class TraceSpansQueryRunner(TraceSpansQueryRunnerMixin, AnalyticsQueryRunner[Tramin(if({where_for_start}, timestamp, NULL)) OVER (PARTITION BY trace_id) as trace_start,{attributes},{resource_attributes},- max(if({where_for_start}, duration_nano, NULL)) OVER (PARTITION BY trace_id) as trace_duration,+ dateDiff(+ 'microsecond',+ min(if({where_for_start}, timestamp, NULL)) OVER (PARTITION BY trace_id),+ max(if({where_for_start}, end_time, NULL)) OVER (PARTITION BY trace_id)+ ) * 1000 as trace_duration,{events}FROM posthog.trace_spansWHERE {filters} AND {trace_filter} LIMIT {limit}diff --git a/products/tracing/backend/tests/test_keyset_pagination.py b/products/tracing/backend/tests/test_keyset_pagination.pyindex d1c033ce28e4..6050a63a84e8 100644--- a/products/tracing/backend/tests/test_keyset_pagination.py+++ b/products/tracing/backend/tests/test_keyset_pagination.py@@ -257,6 +257,9 @@ class TestTraceSpansDurationOrdering(_TraceSpansTestBase):# displays the root span, so slowest/fastest must rank by ROOT duration, not the longest child.ROOT_DURATIONS = [10, 5, 8] # seconds, per trace i=0,1,2 → root order desc = [0, 2, 1]CHILD_DURATIONS = [1, 50, 2] # trace 1's child is the longest span anywhere+MULTI_ROOT_TRACE = 3+MULTI_ROOT_OFFSETS = [0, 20]+MULTI_ROOT_DURATION = 3class TestTraceSpansDurationRootScope(_TraceSpansTestBase):@@ -286,6 +289,15 @@ class TestTraceSpansDurationRootScope(_TraceSpansTestBase):f"('019e8755-0000-0000-0000-{len(rows):012d}', {cls.team.id}, '{trace_id}', '{child_span_id}', "f"'{root_span_id}', 'db query', 2, '{ts_str}', '{child_end}', '{ts_str}', 0, 'web')")+ for n, offset_s in enumerate(MULTI_ROOT_OFFSETS):+ start = base + dt.timedelta(seconds=offset_s)+ end = start + dt.timedelta(seconds=MULTI_ROOT_DURATION)+ rows.append(+ f"('019e8755-0000-0000-0000-{len(rows):012d}', {cls.team.id}, "+ f"'{_b64(MULTI_ROOT_TRACE.to_bytes(16, 'big'))}', '{_b64((2000 + n).to_bytes(8, 'big'))}', '', "+ f"'agent turn', 2, '{start.strftime('%Y-%m-%d %H:%M:%S.%f')}', "+ f"'{end.strftime('%Y-%m-%d %H:%M:%S.%f')}', '{ts_str}', 0, 'agent')"+ )sync_execute("INSERT INTO trace_spans (uuid, team_id, trace_id, span_id, parent_span_id, name, kind, ""timestamp, end_time, observed_timestamp, status_code, service_name) VALUES " + ",".join(rows)@@ -295,12 +307,13 @@ class TestTraceSpansDurationRootScope(_TraceSpansTestBase):[# Root durations [10, 5, 8] → DESC = [0, 2, 1], ASC = [1, 2, 0]. Trace 1's 50s child must# NOT change the rank (that would be ranking by whole-trace max, not the displayed root).- ("slowest_by_root", "DESC", [0, 2, 1]),- ("fastest_by_root", "ASC", [1, 2, 0]),+ ("slowest_by_root", "DESC", 10, [3, 0, 2, 1]),+ ("fastest_by_root", "ASC", 10, [1, 2, 0, 3]),+ ("slowest_selected_first", "DESC", 1, [3]),])- def test_ranks_by_root_span_duration(self, _name, order_direction, expected):- self.assertEqual(self._ordered_trace_indices(order_direction=order_direction, limit=10), expected)+ def test_ranks_by_root_span_duration(self, _name, order_direction, limit, expected):+ self.assertEqual(self._ordered_trace_indices(order_direction=order_direction, limit=limit), expected)# Trace 1 started first but owns the newest span; trace 4 never got a root; trace 5 has 15 spans.diff --git a/products/tracing/frontend/TracingScene.tsx b/products/tracing/frontend/TracingScene.tsxindex 18370ab20d74..7b60418216ba 100644--- a/products/tracing/frontend/TracingScene.tsx+++ b/products/tracing/frontend/TracingScene.tsx@@ -346,6 +346,7 @@ function TracingSceneContents(): JSX.Element {dataSource={listRows}spanColumns={spanColumns}showRootTag={filters.viewMode === 'spans'}+ showTraceDuration={filters.viewMode === 'traces'}loading={spansLoading}hasMoreToLoad={hasMoreToLoad}onLoadMore={fetchNextPage}diff --git a/products/tracing/frontend/components/VirtualizedSpanList/VirtualizedSpanList.tsx b/products/tracing/frontend/components/VirtualizedSpanList/VirtualizedSpanList.tsxindex 97b854b76041..8e59a02a9894 100644--- a/products/tracing/frontend/components/VirtualizedSpanList/VirtualizedSpanList.tsx+++ b/products/tracing/frontend/components/VirtualizedSpanList/VirtualizedSpanList.tsx@@ -13,6 +13,7 @@ import { cn } from 'lib/utils/css-classes'import { TRACING_DATE_FORMAT, TRACING_DISPLAY_TIMEZONE, TRACING_TIME_FORMAT } from '../../dateFormats'import type { ErrorScope } from '../../errorCorrelation'import type { SpanErrorBadge } from '../../spanErrorsLogic'+import { traceRowDuration } from '../../traceRows'import { formatDuration } from '../../TraceWaterfallView'import type { TracingOrderBy, TracingOrderDirection } from '../../tracingFiltersLogic'import { SPAN_KIND_LABELS, STATUS_CODE_LABELS } from '../../types'@@ -65,6 +66,7 @@ interface VirtualizedSpanListProps extends SortProps {spanColumns: SpanColumnConfig[]// Tag root spans. Only useful when the list mixes roots and children.showRootTag?: boolean+ showTraceDuration?: boolean}interface SpanRowProps {@@ -74,6 +76,7 @@ interface SpanRowProps {onRowClick: (span: Span) => voidspanErrors: SpanErrors | undefinedshowRootTag: boolean+ showTraceDuration: boolean}/** Header cell wired to the shared resize handle. `sort` marks the column as server-sortable. */@@ -162,7 +165,12 @@ function SpanRowHeader({)}-function spanCellContent(column: SpanColumnConfig, span: Span, showRootTag: boolean): JSX.Element | null {+function spanCellContent(+ column: SpanColumnConfig,+ span: Span,+ showRootTag: boolean,+ showTraceDuration: boolean+): JSX.Element | null {switch (column.type) {case 'timestamp':return (@@ -199,7 +207,7 @@ function spanCellContent(column: SpanColumnConfig, span: Span, showRootTag: boolcase 'kind':return <>{SPAN_KIND_LABELS[span.kind] ?? span.kind}</>case 'duration':- return <>{formatDuration(span.duration_nano)}</>+ return <>{formatDuration(showTraceDuration ? traceRowDuration(span) : span.duration_nano)}</>case 'status': {const status = STATUS_CODE_LABELS[span.status_code] ?? {label: String(span.status_code),@@ -227,6 +235,7 @@ function SpanRow({widths,spanErrors,showRootTag,+ showTraceDuration,onClick,}: {span: Span@@ -234,6 +243,7 @@ function SpanRow({widths: Record<string, number>spanErrors: SpanErrors | undefinedshowRootTag: boolean+ showTraceDuration: booleanonClick: () => void}): JSX.Element {const errorBadge = spanErrors?.badges.get(span.uuid)@@ -259,7 +269,7 @@ function SpanRow({const key = spanColumnKey(column)return (<TableCell key={key} width={widths[key]}>- {spanCellContent(column, span, showRootTag)}+ {spanCellContent(column, span, showRootTag, showTraceDuration)}</TableCell>)})}@@ -291,6 +301,7 @@ function SpanListRow({widths,spanErrors,showRootTag,+ showTraceDuration,onRowClick,}: {ariaAttributes: { 'aria-posinset': number; 'aria-setsize': number; role: 'listitem' }@@ -308,6 +319,7 @@ function SpanListRow({widths={widths}spanErrors={spanErrors}showRootTag={showRootTag}+ showTraceDuration={showTraceDuration}onClick={() => onRowClick(span)}/></div>@@ -325,6 +337,7 @@ export function VirtualizedSpanList({spanErrors,spanColumns,showRootTag = false,+ showTraceDuration = false,orderBy,orderDirection,onSort,@@ -404,7 +417,15 @@ export function VirtualizedSpanList({rowCount={dataSource.length}rowHeight={ROW_HEIGHT}rowComponent={SpanListRow}- rowProps={{ dataSource, spanColumns, widths, spanErrors, showRootTag, onRowClick }}+ rowProps={{+ dataSource,+ spanColumns,+ widths,+ spanErrors,+ showRootTag,+ showTraceDuration,+ onRowClick,+ }}onRowsRendered={handleRowsRendered}listRef={listRef}/>diff --git a/products/tracing/frontend/traceRows.test.ts b/products/tracing/frontend/traceRows.test.tsindex 54eb497fe938..13f1be26b1e3 100644--- a/products/tracing/frontend/traceRows.test.ts+++ b/products/tracing/frontend/traceRows.test.ts@@ -60,6 +60,18 @@ describe('traceRows', () => {expect(rows.map((s) => s.span_id)).toEqual(expected)})+ it('shows a trace with several root spans as one row, in trace order', () => {+ const laterRootA = span({+ trace_id: 'a',+ span_id: 'a9',+ is_root_span: true,+ timestamp: '2026-06-02T08:00:05.000Z',+ trace_start: '2026-06-02T08:00:00.000Z',+ })+ const rows = traceRows([laterRootA, rootC, rootA], 'timestamp', 'DESC')+ expect(rows.map((s) => s.span_id)).toEqual(['c1', 'a1'])+ })+it('orders by trace duration under duration sort', () => {const rows = traceRows([diff --git a/products/tracing/frontend/traceRows.ts b/products/tracing/frontend/traceRows.tsindex 64f5dfc5dc03..66a86c6d77cd 100644--- a/products/tracing/frontend/traceRows.ts+++ b/products/tracing/frontend/traceRows.ts@@ -6,17 +6,33 @@ import type { Span } from './types'// One row per trace. If the root span never arrived (e.g. an upstream service started the trace// but doesn't export to PostHog), the top loaded span stands in for it.export function traceRows(spans: Span[], orderBy: TracingOrderBy, orderDirection: TracingOrderDirection): Span[] {- const roots = spans.filter((s) => s.is_root_span)- const tracesWithRoot = new Set(roots.map((s) => s.trace_id))+ const rootByTrace = new Map<string, Span>()+ let hasMultiRootTrace = false+ for (const span of spans) {+ if (!span.is_root_span) {+ continue+ }+ const current = rootByTrace.get(span.trace_id)+ if (current === undefined) {+ rootByTrace.set(span.trace_id, span)+ } else {+ hasMultiRootTrace = true+ if (dayjs(span.timestamp).isBefore(current.timestamp)) {+ rootByTrace.set(span.trace_id, span)+ }+ }+ }+ const roots = [...rootByTrace.values()]+const orphans = new Map<string, Span[]>()for (const span of spans) {- if (!tracesWithRoot.has(span.trace_id)) {+ if (!rootByTrace.has(span.trace_id)) {const traceSpans = orphans.get(span.trace_id) ?? []traceSpans.push(span)orphans.set(span.trace_id, traceSpans)}}- if (orphans.size === 0) {+ if (orphans.size === 0 && !hasMultiRootTrace) {return roots}@@ -29,9 +45,13 @@ export function traceRows(spans: Span[], orderBy: TracingOrderBy, orderDirectionreturn { ...top, root_missing: true }})- // The API sorts root rows first, so put promoted rows back in trace order.+ // The API sorts by span, not trace, so restore trace order.const key = (s: Span): number =>- orderBy === 'duration' ? (s.trace_duration ?? s.duration_nano) : dayjs(s.trace_start ?? s.timestamp).valueOf()+ orderBy === 'duration' ? traceRowDuration(s) : dayjs(s.trace_start ?? s.timestamp).valueOf()const dir = orderDirection === 'ASC' ? 1 : -1return [...roots, ...promoted].sort((a, b) => (key(a) - key(b)) * dir)}++export function traceRowDuration(span: Span): number {+ return span.trace_duration ?? span.duration_nano+}