Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(tracing): show multi-root traces as one row with full duration (#128139)

Frank Hamand552bf52c7852
diff --git a/products/tracing/backend/logic.py b/products/tracing/backend/logic.py
index 8ed44dbdd6b5..fb7558af787c 100644
--- a/products/tracing/backend/logic.py
+++ b/products/tracing/backend/logic.py
@@ -63,6 +63,8 @@ TIME_BUCKET_DATE_RANGE_WHERE = (
# before the range, so the trace does not show as having no root.
ROOT_SPAN_LOOKBACK = dt.timedelta(minutes=15)
+TRACE_DURATION_NANO_SQL = "dateDiff('microsecond', min(timestamp), max(end_time)) * 1000"
+
# Hard cap on number of rows returned per period by the span aggregation runners. Keeps
# payloads bounded when name cardinality blows up (e.g. untemplated URL paths). The flame
# graph collapses long tails anyway so the lower-ranked rows aren't visible.
@@ -668,9 +670,9 @@ class TraceSpansQueryRunner(TraceSpansQueryRunnerMixin, AnalyticsQueryRunner[Tra
# per-trace value — `LIMIT 1 BY trace_id` can't paginate cleanly because a multi-span trace
# straddling the boundary would be re-selected via its other spans. Time sorts order by each
# trace's start time (`min(timestamp)`) and keyset on it; duration sorts order by trace
- # duration (`max(duration_nano)`) and offset-paginate (the time index can't prune a duration
- # order, so keyset would pay its cost for none of its benefit).
- sort_key_sql = "max(duration_nano)" if by_duration else "min(timestamp)"
+ # duration (first span start to last span end) and offset-paginate (the time index can't prune
+ # a duration order, so keyset would pay its cost for none of its benefit).
+ sort_key_sql = TRACE_DURATION_NANO_SQL if by_duration else "min(timestamp)"
# rootSpans is gated on `is True` (not truthiness), so None leaves the prefetch-driven
# waterfall untouched. The list endpoint defaults it to True, and the span list sends False
@@ -757,7 +759,7 @@ class TraceSpansQueryRunner(TraceSpansQueryRunnerMixin, AnalyticsQueryRunner[Tra
# `trace_start` / `trace_duration` are the per-trace keys the view paginates and re-sorts on.
# They MUST aggregate over the same rows the trace-selection subquery grouped, or the keys
# diverge from the set the subquery picked — e.g. under root_only the subquery orders traces
- # by `max(duration_nano)` over root spans, so a window over *all* spans would rank a trace by
+ # by the duration of their root spans, so a window over *all* spans would rank a trace by
# a long child the subquery never considered, corrupting the order and offset page boundaries.
# Scope the key windows to match the subquery (cursor/time bounds are deliberately excluded —
# the key is a property of the whole trace, not of the current page).
@@ -817,7 +819,11 @@ class TraceSpansQueryRunner(TraceSpansQueryRunnerMixin, AnalyticsQueryRunner[Tra
min(if({where_for_start}, timestamp, NULL)) OVER (PARTITION BY trace_id) as trace_start,
{attributes},
{resource_attributes},
- max(if({where_for_start}, duration_nano, NULL)) OVER (PARTITION BY trace_id) as trace_duration,
+ dateDiff(
+ 'microsecond',
+ min(if({where_for_start}, timestamp, NULL)) OVER (PARTITION BY trace_id),
+ max(if({where_for_start}, end_time, NULL)) OVER (PARTITION BY trace_id)
+ ) * 1000 as trace_duration,
{events}
FROM posthog.trace_spans
WHERE {filters} AND {trace_filter} LIMIT {limit}
diff --git a/products/tracing/backend/tests/test_keyset_pagination.py b/products/tracing/backend/tests/test_keyset_pagination.py
index d1c033ce28e4..6050a63a84e8 100644
--- a/products/tracing/backend/tests/test_keyset_pagination.py
+++ b/products/tracing/backend/tests/test_keyset_pagination.py
@@ -257,6 +257,9 @@ class TestTraceSpansDurationOrdering(_TraceSpansTestBase):
# displays the root span, so slowest/fastest must rank by ROOT duration, not the longest child.
ROOT_DURATIONS = [10, 5, 8] # seconds, per trace i=0,1,2 → root order desc = [0, 2, 1]
CHILD_DURATIONS = [1, 50, 2] # trace 1's child is the longest span anywhere
+MULTI_ROOT_TRACE = 3
+MULTI_ROOT_OFFSETS = [0, 20]
+MULTI_ROOT_DURATION = 3
class TestTraceSpansDurationRootScope(_TraceSpansTestBase):
@@ -286,6 +289,15 @@ class TestTraceSpansDurationRootScope(_TraceSpansTestBase):
f"('019e8755-0000-0000-0000-{len(rows):012d}', {cls.team.id}, '{trace_id}', '{child_span_id}', "
f"'{root_span_id}', 'db query', 2, '{ts_str}', '{child_end}', '{ts_str}', 0, 'web')"
)
+ for n, offset_s in enumerate(MULTI_ROOT_OFFSETS):
+ start = base + dt.timedelta(seconds=offset_s)
+ end = start + dt.timedelta(seconds=MULTI_ROOT_DURATION)
+ rows.append(
+ f"('019e8755-0000-0000-0000-{len(rows):012d}', {cls.team.id}, "
+ f"'{_b64(MULTI_ROOT_TRACE.to_bytes(16, 'big'))}', '{_b64((2000 + n).to_bytes(8, 'big'))}', '', "
+ f"'agent turn', 2, '{start.strftime('%Y-%m-%d %H:%M:%S.%f')}', "
+ f"'{end.strftime('%Y-%m-%d %H:%M:%S.%f')}', '{ts_str}', 0, 'agent')"
+ )
sync_execute(
"INSERT INTO trace_spans (uuid, team_id, trace_id, span_id, parent_span_id, name, kind, "
"timestamp, end_time, observed_timestamp, status_code, service_name) VALUES " + ",".join(rows)
@@ -295,12 +307,13 @@ class TestTraceSpansDurationRootScope(_TraceSpansTestBase):
[
# Root durations [10, 5, 8] → DESC = [0, 2, 1], ASC = [1, 2, 0]. Trace 1's 50s child must
# NOT change the rank (that would be ranking by whole-trace max, not the displayed root).
- ("slowest_by_root", "DESC", [0, 2, 1]),
- ("fastest_by_root", "ASC", [1, 2, 0]),
+ ("slowest_by_root", "DESC", 10, [3, 0, 2, 1]),
+ ("fastest_by_root", "ASC", 10, [1, 2, 0, 3]),
+ ("slowest_selected_first", "DESC", 1, [3]),
]
)
- def test_ranks_by_root_span_duration(self, _name, order_direction, expected):
- self.assertEqual(self._ordered_trace_indices(order_direction=order_direction, limit=10), expected)
+ def test_ranks_by_root_span_duration(self, _name, order_direction, limit, expected):
+ self.assertEqual(self._ordered_trace_indices(order_direction=order_direction, limit=limit), expected)
# Trace 1 started first but owns the newest span; trace 4 never got a root; trace 5 has 15 spans.
diff --git a/products/tracing/frontend/TracingScene.tsx b/products/tracing/frontend/TracingScene.tsx
index 18370ab20d74..7b60418216ba 100644
--- a/products/tracing/frontend/TracingScene.tsx
+++ b/products/tracing/frontend/TracingScene.tsx
@@ -346,6 +346,7 @@ function TracingSceneContents(): JSX.Element {
dataSource={listRows}
spanColumns={spanColumns}
showRootTag={filters.viewMode === 'spans'}
+ showTraceDuration={filters.viewMode === 'traces'}
loading={spansLoading}
hasMoreToLoad={hasMoreToLoad}
onLoadMore={fetchNextPage}
diff --git a/products/tracing/frontend/components/VirtualizedSpanList/VirtualizedSpanList.tsx b/products/tracing/frontend/components/VirtualizedSpanList/VirtualizedSpanList.tsx
index 97b854b76041..8e59a02a9894 100644
--- a/products/tracing/frontend/components/VirtualizedSpanList/VirtualizedSpanList.tsx
+++ b/products/tracing/frontend/components/VirtualizedSpanList/VirtualizedSpanList.tsx
@@ -13,6 +13,7 @@ import { cn } from 'lib/utils/css-classes'
import { TRACING_DATE_FORMAT, TRACING_DISPLAY_TIMEZONE, TRACING_TIME_FORMAT } from '../../dateFormats'
import type { ErrorScope } from '../../errorCorrelation'
import type { SpanErrorBadge } from '../../spanErrorsLogic'
+import { traceRowDuration } from '../../traceRows'
import { formatDuration } from '../../TraceWaterfallView'
import type { TracingOrderBy, TracingOrderDirection } from '../../tracingFiltersLogic'
import { SPAN_KIND_LABELS, STATUS_CODE_LABELS } from '../../types'
@@ -65,6 +66,7 @@ interface VirtualizedSpanListProps extends SortProps {
spanColumns: SpanColumnConfig[]
// Tag root spans. Only useful when the list mixes roots and children.
showRootTag?: boolean
+ showTraceDuration?: boolean
}
interface SpanRowProps {
@@ -74,6 +76,7 @@ interface SpanRowProps {
onRowClick: (span: Span) => void
spanErrors: SpanErrors | undefined
showRootTag: boolean
+ showTraceDuration: boolean
}
/** Header cell wired to the shared resize handle. `sort` marks the column as server-sortable. */
@@ -162,7 +165,12 @@ function SpanRowHeader({
)
}
-function spanCellContent(column: SpanColumnConfig, span: Span, showRootTag: boolean): JSX.Element | null {
+function spanCellContent(
+ column: SpanColumnConfig,
+ span: Span,
+ showRootTag: boolean,
+ showTraceDuration: boolean
+): JSX.Element | null {
switch (column.type) {
case 'timestamp':
return (
@@ -199,7 +207,7 @@ function spanCellContent(column: SpanColumnConfig, span: Span, showRootTag: bool
case 'kind':
return <>{SPAN_KIND_LABELS[span.kind] ?? span.kind}</>
case 'duration':
- return <>{formatDuration(span.duration_nano)}</>
+ return <>{formatDuration(showTraceDuration ? traceRowDuration(span) : span.duration_nano)}</>
case 'status': {
const status = STATUS_CODE_LABELS[span.status_code] ?? {
label: String(span.status_code),
@@ -227,6 +235,7 @@ function SpanRow({
widths,
spanErrors,
showRootTag,
+ showTraceDuration,
onClick,
}: {
span: Span
@@ -234,6 +243,7 @@ function SpanRow({
widths: Record<string, number>
spanErrors: SpanErrors | undefined
showRootTag: boolean
+ showTraceDuration: boolean
onClick: () => void
}): JSX.Element {
const errorBadge = spanErrors?.badges.get(span.uuid)
@@ -259,7 +269,7 @@ function SpanRow({
const key = spanColumnKey(column)
return (
<TableCell key={key} width={widths[key]}>
- {spanCellContent(column, span, showRootTag)}
+ {spanCellContent(column, span, showRootTag, showTraceDuration)}
</TableCell>
)
})}
@@ -291,6 +301,7 @@ function SpanListRow({
widths,
spanErrors,
showRootTag,
+ showTraceDuration,
onRowClick,
}: {
ariaAttributes: { 'aria-posinset': number; 'aria-setsize': number; role: 'listitem' }
@@ -308,6 +319,7 @@ function SpanListRow({
widths={widths}
spanErrors={spanErrors}
showRootTag={showRootTag}
+ showTraceDuration={showTraceDuration}
onClick={() => onRowClick(span)}
/>
</div>
@@ -325,6 +337,7 @@ export function VirtualizedSpanList({
spanErrors,
spanColumns,
showRootTag = false,
+ showTraceDuration = false,
orderBy,
orderDirection,
onSort,
@@ -404,7 +417,15 @@ export function VirtualizedSpanList({
rowCount={dataSource.length}
rowHeight={ROW_HEIGHT}
rowComponent={SpanListRow}
- rowProps={{ dataSource, spanColumns, widths, spanErrors, showRootTag, onRowClick }}
+ rowProps={{
+ dataSource,
+ spanColumns,
+ widths,
+ spanErrors,
+ showRootTag,
+ showTraceDuration,
+ onRowClick,
+ }}
onRowsRendered={handleRowsRendered}
listRef={listRef}
/>
diff --git a/products/tracing/frontend/traceRows.test.ts b/products/tracing/frontend/traceRows.test.ts
index 54eb497fe938..13f1be26b1e3 100644
--- a/products/tracing/frontend/traceRows.test.ts
+++ b/products/tracing/frontend/traceRows.test.ts
@@ -60,6 +60,18 @@ describe('traceRows', () => {
expect(rows.map((s) => s.span_id)).toEqual(expected)
})
+ it('shows a trace with several root spans as one row, in trace order', () => {
+ const laterRootA = span({
+ trace_id: 'a',
+ span_id: 'a9',
+ is_root_span: true,
+ timestamp: '2026-06-02T08:00:05.000Z',
+ trace_start: '2026-06-02T08:00:00.000Z',
+ })
+ const rows = traceRows([laterRootA, rootC, rootA], 'timestamp', 'DESC')
+ expect(rows.map((s) => s.span_id)).toEqual(['c1', 'a1'])
+ })
+
it('orders by trace duration under duration sort', () => {
const rows = traceRows(
[
diff --git a/products/tracing/frontend/traceRows.ts b/products/tracing/frontend/traceRows.ts
index 64f5dfc5dc03..66a86c6d77cd 100644
--- a/products/tracing/frontend/traceRows.ts
+++ b/products/tracing/frontend/traceRows.ts
@@ -6,17 +6,33 @@ import type { Span } from './types'
// One row per trace. If the root span never arrived (e.g. an upstream service started the trace
// but doesn't export to PostHog), the top loaded span stands in for it.
export function traceRows(spans: Span[], orderBy: TracingOrderBy, orderDirection: TracingOrderDirection): Span[] {
- const roots = spans.filter((s) => s.is_root_span)
- const tracesWithRoot = new Set(roots.map((s) => s.trace_id))
+ const rootByTrace = new Map<string, Span>()
+ let hasMultiRootTrace = false
+ for (const span of spans) {
+ if (!span.is_root_span) {
+ continue
+ }
+ const current = rootByTrace.get(span.trace_id)
+ if (current === undefined) {
+ rootByTrace.set(span.trace_id, span)
+ } else {
+ hasMultiRootTrace = true
+ if (dayjs(span.timestamp).isBefore(current.timestamp)) {
+ rootByTrace.set(span.trace_id, span)
+ }
+ }
+ }
+ const roots = [...rootByTrace.values()]
+
const orphans = new Map<string, Span[]>()
for (const span of spans) {
- if (!tracesWithRoot.has(span.trace_id)) {
+ if (!rootByTrace.has(span.trace_id)) {
const traceSpans = orphans.get(span.trace_id) ?? []
traceSpans.push(span)
orphans.set(span.trace_id, traceSpans)
}
}
- if (orphans.size === 0) {
+ if (orphans.size === 0 && !hasMultiRootTrace) {
return roots
}
@@ -29,9 +45,13 @@ export function traceRows(spans: Span[], orderBy: TracingOrderBy, orderDirection
return { ...top, root_missing: true }
})
- // The API sorts root rows first, so put promoted rows back in trace order.
+ // The API sorts by span, not trace, so restore trace order.
const key = (s: Span): number =>
- orderBy === 'duration' ? (s.trace_duration ?? s.duration_nano) : dayjs(s.trace_start ?? s.timestamp).valueOf()
+ orderBy === 'duration' ? traceRowDuration(s) : dayjs(s.trace_start ?? s.timestamp).valueOf()
const dir = orderDirection === 'ASC' ? 1 : -1
return [...roots, ...promoted].sort((a, b) => (key(a) - key(b)) * dir)
}
+
+export function traceRowDuration(span: Span): number {
+ return span.trace_duration ?? span.duration_nano
+}