Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(topics): show a trace's topics in a trace header hover card (#19238)

Lysander Kieself96e9bfb2ea8
diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.ts
index c6d8d1d72a..9eec1ebcb0 100644
--- a/packages/shared/src/server/topics/clickhouse.ts
+++ b/packages/shared/src/server/topics/clickhouse.ts
@@ -385,9 +385,11 @@ export async function getLatestFacetSummaries(
facetId: string,
facetVersion: number | undefined,
timeRange: TopicTimeRange,
+ traceIds?: string[],
) {
const scopeSql = `facet_id = {facetId:String}
- ${facetVersion ? "AND facet_version = {facetVersion:UInt32}" : ""}`;
+ ${facetVersion ? "AND facet_version = {facetVersion:UInt32}" : ""}
+ ${traceIds ? "AND trace_id IN {traceIds:Array(String)}" : ""}`;
const rows = await queryClickhouse<
Pick<
TopicSummary,
@@ -416,6 +418,7 @@ export async function getLatestFacetSummaries(
...timeRangeParams(timeRange),
facetId,
...(facetVersion ? { facetVersion } : {}),
+ ...(traceIds ? { traceIds } : {}),
},
tags: { route: "topics-latest-summaries", projectId },
});
@@ -944,6 +947,7 @@ export async function readLatestTopicAssignments(
projectId: string,
facetId: string,
timeRange: TopicTimeRange,
+ traceIds?: string[],
): Promise<TopicAssignment[]> {
const publishedRuns = await prisma.topicClusteringRun.findMany({
where: {
@@ -957,6 +961,7 @@ export async function readLatestTopicAssignments(
query: `SELECT ${assignmentColumns} FROM topic_assignments
WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND facet_id = {facetId:String}
AND (clustering_run_id = '' OR clustering_run_id IN ({publishedRunIds:Array(String)}))
+ ${traceIds ? "AND trace_id IN {traceIds:Array(String)}" : ""}
ORDER BY facet_version DESC, assigned_at DESC, clustering_run_id DESC, origin DESC
LIMIT 1 BY project_id, facet_id, trace_id, if(trace_id = '', session_id, '')`,
params: {
@@ -964,8 +969,35 @@ export async function readLatestTopicAssignments(
...timeRangeParams(timeRange),
facetId,
publishedRunIds: publishedRuns.map(({ id }) => id),
+ ...(traceIds ? { traceIds } : {}),
},
tags: { route: "topics-current-assignments", projectId },
});
return rows.map(assignmentResult);
}
+
+/** Source time span of one trace's summaries, served by the latest_source_versions projection; null when never summarized. */
+export async function getTopicTraceUnitTimeRange(
+ projectId: string,
+ traceId: string,
+ timeRange: TopicTimeRange,
+): Promise<TopicTimeRange | null> {
+ const [row] = await queryClickhouse<{
+ found: string;
+ fromMs: string;
+ toMs: string;
+ }>({
+ query: `SELECT count() AS found,
+ toUnixTimestamp64Milli(min(unit_start_time)) AS fromMs,
+ toUnixTimestamp64Milli(max(unit_start_time)) AS toMs
+ FROM topic_facet_summaries
+ WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND trace_id = {traceId:String}`,
+ params: { projectId, ...timeRangeParams(timeRange), traceId },
+ tags: { route: "topics-trace-unit-time", projectId },
+ });
+ if (!row || Number(row.found) === 0) return null;
+ return {
+ from: new Date(Number(row.fromMs)),
+ to: new Date(Number(row.toMs) + 1),
+ };
+}
diff --git a/web/src/__tests__/server/unit/topics.servertest.ts b/web/src/__tests__/server/unit/topics.servertest.ts
index 80a46a4c0f..8ec369bcf2 100644
--- a/web/src/__tests__/server/unit/topics.servertest.ts
+++ b/web/src/__tests__/server/unit/topics.servertest.ts
@@ -1299,11 +1299,13 @@ describe("Topics current results", () => {
"intent",
undefined,
timeRange,
+ undefined,
);
expect(mocks.readLatestTopicAssignments).toHaveBeenCalledWith(
projectId,
"intent",
timeRange,
+ undefined,
);
await expect(caller("VIEWER").trigger(input)).rejects.toMatchObject({
code: "FORBIDDEN",
@@ -1403,6 +1405,7 @@ describe("Topics current results", () => {
projectId,
"intent",
timeRange,
+ undefined,
);
expect(mocks.getTopicRuns).toHaveBeenCalledWith(projectId, []);
expect(result[0]).toMatchObject({
diff --git a/web/src/components/design-system/Badge/Badge.stories.tsx b/web/src/components/design-system/Badge/Badge.stories.tsx
index 91ce9f2451..0bc4370b66 100644
--- a/web/src/components/design-system/Badge/Badge.stories.tsx
+++ b/web/src/components/design-system/Badge/Badge.stories.tsx
@@ -37,6 +37,9 @@ const allColors = Object.keys({
orangeTint: true,
limeTint: true,
fuchsiaTint: true,
+ violetHighlight: true,
+ blueHighlight: true,
+ amberHighlight: true,
inverted: true,
} satisfies Record<Color, true>) as Color[];
diff --git a/web/src/components/design-system/Badge/Badge.tsx b/web/src/components/design-system/Badge/Badge.tsx
index 3a923433cb..000e9156a7 100644
--- a/web/src/components/design-system/Badge/Badge.tsx
+++ b/web/src/components/design-system/Badge/Badge.tsx
@@ -34,6 +34,13 @@ const badgeVariants = cva(
/** `filled` for a chip on a muted or accent surface, e.g. a selected or hovered row, where `bg-muted` vanishes. */
filledOnMuted:
"border-transparent bg-background text-foreground-secondary",
+ /** Category highlight tints, as used by treemap tiles. */
+ violetHighlight:
+ "border-transparent bg-category-violet-highlight/35 text-foreground-secondary",
+ blueHighlight:
+ "border-transparent bg-category-blue-highlight/35 text-foreground-secondary",
+ amberHighlight:
+ "border-transparent bg-category-amber-highlight/35 text-foreground-secondary",
violetTint:
"border-category-violet/40 bg-category-violet/10 text-foreground-secondary",
skyTint:
diff --git a/web/src/features/topics/server/currentResults.ts b/web/src/features/topics/server/currentResults.ts
index 794dd03ac5..1df0247459 100644
--- a/web/src/features/topics/server/currentResults.ts
+++ b/web/src/features/topics/server/currentResults.ts
@@ -3,10 +3,12 @@ import type {
TopicSummary,
TopicTimeRange,
} from "@langfuse/shared/topics";
+import { OBSERVATIONS_TO_TRACE_INTERVAL_DAYS } from "@langfuse/shared/src/server";
import {
getLatestFacetSummaries,
getPublishedTopicRun,
getTopicRuns,
+ getTopicTraceUnitTimeRange,
hasStoredTopicSummaries,
listTopicFacets,
listTopicRules,
@@ -46,6 +48,7 @@ function resolveTopicResult(
export async function currentTopicResults(
projectId: string,
timeRange: TopicTimeRange,
+ traceIds?: string[],
) {
const facets = await listTopicFacets(projectId);
const hasStoredSummaries =
@@ -60,8 +63,14 @@ export async function currentTopicResults(
.map(async (facet) => {
const version = facet.versions[0];
const [storedAssignments, storedSummaries, run] = await Promise.all([
- readLatestTopicAssignments(projectId, facet.id, timeRange),
- getLatestFacetSummaries(projectId, facet.id, undefined, timeRange),
+ readLatestTopicAssignments(projectId, facet.id, timeRange, traceIds),
+ getLatestFacetSummaries(
+ projectId,
+ facet.id,
+ undefined,
+ timeRange,
+ traceIds,
+ ),
getPublishedTopicRun(projectId, facet.id),
]);
const latestSummaries = storedSummaries.filter(
@@ -176,3 +185,35 @@ export async function currentTopicResults(
);
return { hasStoredSummaries, facets: currentFacets };
}
+
+/** Current result per facet for one trace; summaries start within a few days of the trace. */
+export async function traceFacetResults(
+ projectId: string,
+ traceId: string,
+ timestamp: Date,
+) {
+ const slackMs = OBSERVATIONS_TO_TRACE_INTERVAL_DAYS * 86_400_000;
+ // Narrow to the trace's exact source time so assignment reads prune to its minute.
+ const unitTimeRange = await getTopicTraceUnitTimeRange(projectId, traceId, {
+ from: new Date(timestamp.getTime() - slackMs),
+ to: new Date(timestamp.getTime() + slackMs),
+ });
+ if (!unitTimeRange) return [];
+ const { facets } = await currentTopicResults(projectId, unitTimeRange, [
+ traceId,
+ ]);
+ return facets.map((facet) => {
+ const row = facet.rows.find((item) => item.traceId === traceId);
+ return {
+ facetId: facet.facetId,
+ name: facet.name,
+ summary: row?.summary ?? null,
+ outcome: row?.outcome ?? null,
+ topicId: row?.topicId ?? null,
+ topicName: row?.topicName ?? null,
+ // Slot in the published map; picks the topic's color.
+ topicIndex:
+ facet.map?.topics.findIndex((topic) => topic.id === row?.topicId) ?? -1,
+ };
+ });
+}
diff --git a/web/src/features/topics/server/topicsRouter.ts b/web/src/features/topics/server/topicsRouter.ts
index 938c7a22d9..a0ace71aee 100644
--- a/web/src/features/topics/server/topicsRouter.ts
+++ b/web/src/features/topics/server/topicsRouter.ts
@@ -66,7 +66,7 @@ import {
topicTriggerInputSchema,
} from "./traceSelection";
-import { currentTopicResults } from "./currentResults";
+import { currentTopicResults, traceFacetResults } from "./currentResults";
import { TOPIC_SETUP_BACKFILL_LIMIT } from "../topic-setup-flow";
const projectInput = z.object({ projectId: topicIdSchema });
@@ -320,6 +320,14 @@ export const topicsRouter = createTRPCRouter({
.query(({ input }) =>
currentTopicResults(input.projectId, input.timeRange),
),
+ /** Trace header: every facet result of one trace, with its summary. */
+ forTrace: topicsProcedure
+ .input(
+ projectInput.extend({ traceId: z.string().min(1), timestamp: z.date() }),
+ )
+ .query(({ input }) =>
+ traceFacetResults(input.projectId, input.traceId, input.timestamp),
+ ),
previewTraces: topicsProcedure
.input(topicTraceSelectionSchema)
.query(({ input, ctx }) => previewTopicTraces(input, ctx.prisma)),
diff --git a/web/src/features/topics/topic-map-colors.ts b/web/src/features/topics/topic-map-colors.ts
index 902d6ef18b..929d1a9a2e 100644
--- a/web/src/features/topics/topic-map-colors.ts
+++ b/web/src/features/topics/topic-map-colors.ts
@@ -53,6 +53,14 @@ export const treemapTileClasses = (rank: number, big: boolean) => {
export const treemapSwatchColor = (rank: number) =>
`var(--category-${["violet", "blue", "amber"][Math.max(0, rank) % 3]}-highlight)`;
+/** Topic chip color for a treemap rank; grey when the topic is not in the current results. */
+export const treemapBadgeColor = (rank: number) =>
+ rank < 0
+ ? ("filled" as const)
+ : (["violetHighlight", "blueHighlight", "amberHighlight"] as const)[
+ rank % 3
+ ];
+
/** Grey tile for traces without a topic. */
export const unassignedTileClasses = {
tile: "bg-category-neutral-tint",
diff --git a/web/src/features/topics/trace/TraceTopicsHoverCard.tsx b/web/src/features/topics/trace/TraceTopicsHoverCard.tsx
new file mode 100644
index 0000000000..2e3e130470
--- /dev/null
+++ b/web/src/features/topics/trace/TraceTopicsHoverCard.tsx
@@ -0,0 +1,159 @@
+import { Badge, BadgeShell } from "@/src/components/design-system/Badge/Badge";
+import { HoverCard } from "@/src/components/design-system/HoverCard/HoverCard";
+import { TextLink } from "@/src/components/design-system/TextLink/TextLink";
+import {
+ treemapBadgeColor,
+ treemapSwatchColor,
+} from "@/src/features/topics/topic-map-colors";
+import { useCanViewTopics } from "@/src/features/topics/topicsAccess";
+import { type ReactNode } from "react";
+import { api, type RouterOutputs } from "@/src/utils/api";
+
+type TraceTopicFacet = RouterOutputs["topics"]["forTrace"][number];
+
+/** Facet results of a trace; empty when Topics is unavailable or nothing was summarized. */
+export function useTraceTopicFacets(trace: {
+ projectId: string;
+ id: string;
+ timestamp: Date;
+}) {
+ const canViewTopics = useCanViewTopics(trace.projectId);
+ return (
+ api.topics.forTrace.useQuery(
+ {
+ projectId: trace.projectId,
+ traceId: trace.id,
+ timestamp: trace.timestamp,
+ },
+ { enabled: canViewTopics, staleTime: 60 * 1000 },
+ ).data ?? []
+ );
+}
+
+/** Outcome without a summary, e.g. "not_applicable" as "Not applicable". */
+function outcomeLabel(outcome: string) {
+ const text = outcome.replaceAll("_", " ");
+ return text.charAt(0).toUpperCase() + text.slice(1);
+}
+
+/** Map hue, nudged toward the text color so it reads on the chip tint. */
+function TopicDot({ rank }: { rank: number }) {
+ return (
+ <span
+ aria-hidden
+ className="size-2 shrink-0 rounded-xs"
+ style={{
+ backgroundColor: `color-mix(in oklch, ${treemapSwatchColor(rank)} 80%, hsl(var(--foreground)))`,
+ }}
+ />
+ );
+}
+
+/** Topic chip colored by the topic's slot in the published map. */
+function TopicChip({ name, mapIndex }: { name: string; mapIndex: number }) {
+ return (
+ <span className="flex min-w-0">
+ <BadgeShell color={treemapBadgeColor(mapIndex)}>
+ <TopicDot rank={mapIndex} />
+ <span className="truncate" title={name}>
+ {name}
+ </span>
+ </BadgeShell>
+ </span>
+ );
+}
+
+/** Trace header trigger: the trace's first topic and how many more it has. */
+function TraceTopicsTrigger({
+ facets,
+ triggerProps,
+}: {
+ facets: TraceTopicFacet[];
+ triggerProps: Record<string, unknown>;
+}) {
+ const assigned = facets.filter((facet) => facet.topicName);
+ const lead = assigned[0];
+ return (
+ <span
+ className="ph-no-capture hover:bg-muted inline-flex min-w-0 cursor-pointer rounded-sm px-2"
+ {...triggerProps}
+ >
+ {lead?.topicName ? (
+ <BadgeShell color="ghost" font="mono">
+ <TopicDot rank={lead.topicIndex} />
+ <span className="truncate" title={lead.topicName}>
+ {lead.topicName}
+ </span>
+ {assigned.length > 1 && (
+ <span className="text-muted-foreground shrink-0">
+ +{assigned.length - 1}
+ </span>
+ )}
+ </BadgeShell>
+ ) : (
+ <Badge color="ghost" font="mono" text="Topics" title="" />
+ )}
+ </span>
+ );
+}
+
+/** Card listing every facet result of a trace; the caller renders the trigger. */
+export function TraceTopicsHoverCard({
+ projectId,
+ facets,
+ children,
+}: {
+ projectId: string;
+ facets: TraceTopicFacet[];
+ children: (triggerProps: Record<string, unknown>) => ReactNode;
+}) {
+ return (
+ <HoverCard
+ openDelay={200}
+ placement="bottom-end"
+ content={
+ <div className="ph-no-capture flex max-h-[70vh] w-120 flex-col overflow-y-auto px-4">
+ <div className="flex items-center justify-between gap-2 border-b py-2.5">
+ <span className="text-sm font-bold">Topics</span>
+ <TextLink
+ path={`/project/${encodeURIComponent(projectId)}/topics`}
+ value="Open in Topics →"
+ />
+ </div>
+ {facets.map((facet) => (
+ <div
+ key={facet.facetId}
+ className="flex flex-col gap-1.5 border-b py-3 last:border-b-0"
+ >
+ <div className="flex items-center justify-between gap-3">
+ <span className="text-muted-foreground shrink-0 text-sm">
+ {facet.name}
+ </span>
+ {facet.topicName && (
+ <TopicChip
+ name={facet.topicName}
+ mapIndex={facet.topicIndex}
+ />
+ )}
+ {!facet.summary && (
+ <span className="text-muted-foreground text-sm">
+ {facet.outcome
+ ? outcomeLabel(facet.outcome)
+ : "Not analyzed yet"}
+ </span>
+ )}
+ </div>
+ {facet.summary && (
+ <p className="text-sm text-pretty">{facet.summary}</p>
+ )}
+ </div>
+ ))}
+ </div>
+ }
+ >
+ {({ getTriggerProps }) => children(getTriggerProps())}
+ </HoverCard>
+ );
+}
+
+TraceTopicsHoverCard.Trigger = TraceTopicsTrigger;
diff --git a/web/src/features/traces/components/TraceHeader.tsx b/web/src/features/traces/components/TraceHeader.tsx
index 0afee4cbad..729933ad5f 100644
--- a/web/src/features/traces/components/TraceHeader.tsx
+++ b/web/src/features/traces/components/TraceHeader.tsx
@@ -24,6 +24,10 @@ import { useTraceData } from "@/src/features/traces/contexts/TraceDataProvider";
import { useViewPreferences } from "@/src/features/traces/contexts/ViewPreferencesProvider";
import { aggregateTraceMetrics } from "@/src/features/traces/fns/traceAggregation";
import { cn } from "@/src/utils/tailwind";
+import {
+ TraceTopicsHoverCard,
+ useTraceTopicFacets,
+} from "@/src/features/topics/trace/TraceTopicsHoverCard";
const MAX_VISIBLE_TAGS = 3;
@@ -68,6 +72,7 @@ function TraceHeaderLoading() {
function LoadedTraceHeader() {
const { trace, observations, mergedScores } = useTraceData();
const [showAllTags, setShowAllTags] = useState(false);
+ const topicFacets = useTraceTopicFacets(trace);
const handleShowFewerTags = () => {
setShowAllTags(false);
};
@@ -118,6 +123,19 @@ function LoadedTraceHeader() {
{trace.userId && (
<UserIdBadge userId={trace.userId} projectId={trace.projectId} />
)}
+ {topicFacets.some((facet) => facet.summary) && (
+ <TraceTopicsHoverCard
+ projectId={trace.projectId}
+ facets={topicFacets}
+ >
+ {(triggerProps) => (
+ <TraceTopicsHoverCard.Trigger
+ facets={topicFacets}
+ triggerProps={triggerProps}
+ />
+ )}
+ </TraceTopicsHoverCard>
+ )}
{(traceScores.length > 0 || trace.tags.length > 0) && (
<div className="flex min-w-0 flex-wrap items-center gap-1">
{traceScores.length > 0 && (
diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.ts
index c550326a09..420a80f5d0 100644
--- a/worker/src/features/topics/storage.integration.test.ts
+++ b/worker/src/features/topics/storage.integration.test.ts
@@ -17,6 +17,7 @@ import {
getTopicClusteringSummaryStats,
getTopicSummaryCounts,
getTopicExecutionCosts,
+ getTopicTraceUnitTimeRange,
createTopicFacet,
createTopicFacetVersion,
ensureDefaultTopicFacets,
@@ -654,6 +655,41 @@ describe("Topics definition persistence", () => {
processedAt: newerVersion.processedAt,
},
]);
+ expect(
+ await Promise.all(
+ [[summary.traceId], ["other-trace"]].map((traceIds) =>
+ getLatestFacetSummaries(
+ projectId,
+ "facet",
+ undefined,
+ oldRange,
+ traceIds,
+ ),
+ ),
+ ),
+ ).toEqual([
+ [
+ {
+ ...displaySummary,
+ facetVersion: newerVersion.facetVersion,
+ processedAt: newerVersion.processedAt,
+ },
+ ],
+ [],
+ ]);
+ expect(
+ await Promise.all(
+ [summary.traceId!, "other-trace"].map((traceId) =>
+ getTopicTraceUnitTimeRange(projectId, traceId, oldRange),
+ ),
+ ),
+ ).toEqual([
+ {
+ from: new Date(summary.unitStartTime),
+ to: new Date(Date.parse(summary.unitStartTime) + 1),
+ },
+ null,
+ ]);
expect(
await getLatestFacetSummaries(projectId, "facet", undefined, newRange),
).toEqual([]);