🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(topics): show a trace's topics in a trace header hover card (#19238)
Lysander Kieself96e9bfb2ea8
diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.tsindex c6d8d1d72a..9eec1ebcb0 100644--- a/packages/shared/src/server/topics/clickhouse.ts+++ b/packages/shared/src/server/topics/clickhouse.ts@@ -385,9 +385,11 @@ export async function getLatestFacetSummaries(facetId: string,facetVersion: number | undefined,timeRange: TopicTimeRange,+ traceIds?: string[],) {const scopeSql = `facet_id = {facetId:String}- ${facetVersion ? "AND facet_version = {facetVersion:UInt32}" : ""}`;+ ${facetVersion ? "AND facet_version = {facetVersion:UInt32}" : ""}+ ${traceIds ? "AND trace_id IN {traceIds:Array(String)}" : ""}`;const rows = await queryClickhouse<Pick<TopicSummary,@@ -416,6 +418,7 @@ export async function getLatestFacetSummaries(...timeRangeParams(timeRange),facetId,...(facetVersion ? { facetVersion } : {}),+ ...(traceIds ? { traceIds } : {}),},tags: { route: "topics-latest-summaries", projectId },});@@ -944,6 +947,7 @@ export async function readLatestTopicAssignments(projectId: string,facetId: string,timeRange: TopicTimeRange,+ traceIds?: string[],): Promise<TopicAssignment[]> {const publishedRuns = await prisma.topicClusteringRun.findMany({where: {@@ -957,6 +961,7 @@ export async function readLatestTopicAssignments(query: `SELECT ${assignmentColumns} FROM topic_assignmentsWHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND facet_id = {facetId:String}AND (clustering_run_id = '' OR clustering_run_id IN ({publishedRunIds:Array(String)}))+ ${traceIds ? "AND trace_id IN {traceIds:Array(String)}" : ""}ORDER BY facet_version DESC, assigned_at DESC, clustering_run_id DESC, origin DESCLIMIT 1 BY project_id, facet_id, trace_id, if(trace_id = '', session_id, '')`,params: {@@ -964,8 +969,35 @@ export async function readLatestTopicAssignments(...timeRangeParams(timeRange),facetId,publishedRunIds: publishedRuns.map(({ id }) => id),+ ...(traceIds ? { traceIds } : {}),},tags: { route: "topics-current-assignments", projectId },});return rows.map(assignmentResult);}++/** Source time span of one trace's summaries, served by the latest_source_versions projection; null when never summarized. */+export async function getTopicTraceUnitTimeRange(+ projectId: string,+ traceId: string,+ timeRange: TopicTimeRange,+): Promise<TopicTimeRange | null> {+ const [row] = await queryClickhouse<{+ found: string;+ fromMs: string;+ toMs: string;+ }>({+ query: `SELECT count() AS found,+ toUnixTimestamp64Milli(min(unit_start_time)) AS fromMs,+ toUnixTimestamp64Milli(max(unit_start_time)) AS toMs+ FROM topic_facet_summaries+ WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND trace_id = {traceId:String}`,+ params: { projectId, ...timeRangeParams(timeRange), traceId },+ tags: { route: "topics-trace-unit-time", projectId },+ });+ if (!row || Number(row.found) === 0) return null;+ return {+ from: new Date(Number(row.fromMs)),+ to: new Date(Number(row.toMs) + 1),+ };+}diff --git a/web/src/__tests__/server/unit/topics.servertest.ts b/web/src/__tests__/server/unit/topics.servertest.tsindex 80a46a4c0f..8ec369bcf2 100644--- a/web/src/__tests__/server/unit/topics.servertest.ts+++ b/web/src/__tests__/server/unit/topics.servertest.ts@@ -1299,11 +1299,13 @@ describe("Topics current results", () => {"intent",undefined,timeRange,+ undefined,);expect(mocks.readLatestTopicAssignments).toHaveBeenCalledWith(projectId,"intent",timeRange,+ undefined,);await expect(caller("VIEWER").trigger(input)).rejects.toMatchObject({code: "FORBIDDEN",@@ -1403,6 +1405,7 @@ describe("Topics current results", () => {projectId,"intent",timeRange,+ undefined,);expect(mocks.getTopicRuns).toHaveBeenCalledWith(projectId, []);expect(result[0]).toMatchObject({diff --git a/web/src/components/design-system/Badge/Badge.stories.tsx b/web/src/components/design-system/Badge/Badge.stories.tsxindex 91ce9f2451..0bc4370b66 100644--- a/web/src/components/design-system/Badge/Badge.stories.tsx+++ b/web/src/components/design-system/Badge/Badge.stories.tsx@@ -37,6 +37,9 @@ const allColors = Object.keys({orangeTint: true,limeTint: true,fuchsiaTint: true,+ violetHighlight: true,+ blueHighlight: true,+ amberHighlight: true,inverted: true,} satisfies Record<Color, true>) as Color[];diff --git a/web/src/components/design-system/Badge/Badge.tsx b/web/src/components/design-system/Badge/Badge.tsxindex 3a923433cb..000e9156a7 100644--- a/web/src/components/design-system/Badge/Badge.tsx+++ b/web/src/components/design-system/Badge/Badge.tsx@@ -34,6 +34,13 @@ const badgeVariants = cva(/** `filled` for a chip on a muted or accent surface, e.g. a selected or hovered row, where `bg-muted` vanishes. */filledOnMuted:"border-transparent bg-background text-foreground-secondary",+ /** Category highlight tints, as used by treemap tiles. */+ violetHighlight:+ "border-transparent bg-category-violet-highlight/35 text-foreground-secondary",+ blueHighlight:+ "border-transparent bg-category-blue-highlight/35 text-foreground-secondary",+ amberHighlight:+ "border-transparent bg-category-amber-highlight/35 text-foreground-secondary",violetTint:"border-category-violet/40 bg-category-violet/10 text-foreground-secondary",skyTint:diff --git a/web/src/features/topics/server/currentResults.ts b/web/src/features/topics/server/currentResults.tsindex 794dd03ac5..1df0247459 100644--- a/web/src/features/topics/server/currentResults.ts+++ b/web/src/features/topics/server/currentResults.ts@@ -3,10 +3,12 @@ import type {TopicSummary,TopicTimeRange,} from "@langfuse/shared/topics";+import { OBSERVATIONS_TO_TRACE_INTERVAL_DAYS } from "@langfuse/shared/src/server";import {getLatestFacetSummaries,getPublishedTopicRun,getTopicRuns,+ getTopicTraceUnitTimeRange,hasStoredTopicSummaries,listTopicFacets,listTopicRules,@@ -46,6 +48,7 @@ function resolveTopicResult(export async function currentTopicResults(projectId: string,timeRange: TopicTimeRange,+ traceIds?: string[],) {const facets = await listTopicFacets(projectId);const hasStoredSummaries =@@ -60,8 +63,14 @@ export async function currentTopicResults(.map(async (facet) => {const version = facet.versions[0];const [storedAssignments, storedSummaries, run] = await Promise.all([- readLatestTopicAssignments(projectId, facet.id, timeRange),- getLatestFacetSummaries(projectId, facet.id, undefined, timeRange),+ readLatestTopicAssignments(projectId, facet.id, timeRange, traceIds),+ getLatestFacetSummaries(+ projectId,+ facet.id,+ undefined,+ timeRange,+ traceIds,+ ),getPublishedTopicRun(projectId, facet.id),]);const latestSummaries = storedSummaries.filter(@@ -176,3 +185,35 @@ export async function currentTopicResults();return { hasStoredSummaries, facets: currentFacets };}++/** Current result per facet for one trace; summaries start within a few days of the trace. */+export async function traceFacetResults(+ projectId: string,+ traceId: string,+ timestamp: Date,+) {+ const slackMs = OBSERVATIONS_TO_TRACE_INTERVAL_DAYS * 86_400_000;+ // Narrow to the trace's exact source time so assignment reads prune to its minute.+ const unitTimeRange = await getTopicTraceUnitTimeRange(projectId, traceId, {+ from: new Date(timestamp.getTime() - slackMs),+ to: new Date(timestamp.getTime() + slackMs),+ });+ if (!unitTimeRange) return [];+ const { facets } = await currentTopicResults(projectId, unitTimeRange, [+ traceId,+ ]);+ return facets.map((facet) => {+ const row = facet.rows.find((item) => item.traceId === traceId);+ return {+ facetId: facet.facetId,+ name: facet.name,+ summary: row?.summary ?? null,+ outcome: row?.outcome ?? null,+ topicId: row?.topicId ?? null,+ topicName: row?.topicName ?? null,+ // Slot in the published map; picks the topic's color.+ topicIndex:+ facet.map?.topics.findIndex((topic) => topic.id === row?.topicId) ?? -1,+ };+ });+}diff --git a/web/src/features/topics/server/topicsRouter.ts b/web/src/features/topics/server/topicsRouter.tsindex 938c7a22d9..a0ace71aee 100644--- a/web/src/features/topics/server/topicsRouter.ts+++ b/web/src/features/topics/server/topicsRouter.ts@@ -66,7 +66,7 @@ import {topicTriggerInputSchema,} from "./traceSelection";-import { currentTopicResults } from "./currentResults";+import { currentTopicResults, traceFacetResults } from "./currentResults";import { TOPIC_SETUP_BACKFILL_LIMIT } from "../topic-setup-flow";const projectInput = z.object({ projectId: topicIdSchema });@@ -320,6 +320,14 @@ export const topicsRouter = createTRPCRouter({.query(({ input }) =>currentTopicResults(input.projectId, input.timeRange),),+ /** Trace header: every facet result of one trace, with its summary. */+ forTrace: topicsProcedure+ .input(+ projectInput.extend({ traceId: z.string().min(1), timestamp: z.date() }),+ )+ .query(({ input }) =>+ traceFacetResults(input.projectId, input.traceId, input.timestamp),+ ),previewTraces: topicsProcedure.input(topicTraceSelectionSchema).query(({ input, ctx }) => previewTopicTraces(input, ctx.prisma)),diff --git a/web/src/features/topics/topic-map-colors.ts b/web/src/features/topics/topic-map-colors.tsindex 902d6ef18b..929d1a9a2e 100644--- a/web/src/features/topics/topic-map-colors.ts+++ b/web/src/features/topics/topic-map-colors.ts@@ -53,6 +53,14 @@ export const treemapTileClasses = (rank: number, big: boolean) => {export const treemapSwatchColor = (rank: number) =>`var(--category-${["violet", "blue", "amber"][Math.max(0, rank) % 3]}-highlight)`;+/** Topic chip color for a treemap rank; grey when the topic is not in the current results. */+export const treemapBadgeColor = (rank: number) =>+ rank < 0+ ? ("filled" as const)+ : (["violetHighlight", "blueHighlight", "amberHighlight"] as const)[+ rank % 3+ ];+/** Grey tile for traces without a topic. */export const unassignedTileClasses = {tile: "bg-category-neutral-tint",diff --git a/web/src/features/topics/trace/TraceTopicsHoverCard.tsx b/web/src/features/topics/trace/TraceTopicsHoverCard.tsxnew file mode 100644index 0000000000..2e3e130470--- /dev/null+++ b/web/src/features/topics/trace/TraceTopicsHoverCard.tsx@@ -0,0 +1,159 @@+import { Badge, BadgeShell } from "@/src/components/design-system/Badge/Badge";+import { HoverCard } from "@/src/components/design-system/HoverCard/HoverCard";+import { TextLink } from "@/src/components/design-system/TextLink/TextLink";+import {+ treemapBadgeColor,+ treemapSwatchColor,+} from "@/src/features/topics/topic-map-colors";+import { useCanViewTopics } from "@/src/features/topics/topicsAccess";+import { type ReactNode } from "react";+import { api, type RouterOutputs } from "@/src/utils/api";++type TraceTopicFacet = RouterOutputs["topics"]["forTrace"][number];++/** Facet results of a trace; empty when Topics is unavailable or nothing was summarized. */+export function useTraceTopicFacets(trace: {+ projectId: string;+ id: string;+ timestamp: Date;+}) {+ const canViewTopics = useCanViewTopics(trace.projectId);+ return (+ api.topics.forTrace.useQuery(+ {+ projectId: trace.projectId,+ traceId: trace.id,+ timestamp: trace.timestamp,+ },+ { enabled: canViewTopics, staleTime: 60 * 1000 },+ ).data ?? []+ );+}++/** Outcome without a summary, e.g. "not_applicable" as "Not applicable". */+function outcomeLabel(outcome: string) {+ const text = outcome.replaceAll("_", " ");+ return text.charAt(0).toUpperCase() + text.slice(1);+}++/** Map hue, nudged toward the text color so it reads on the chip tint. */+function TopicDot({ rank }: { rank: number }) {+ return (+ <span+ aria-hidden+ className="size-2 shrink-0 rounded-xs"+ style={{+ backgroundColor: `color-mix(in oklch, ${treemapSwatchColor(rank)} 80%, hsl(var(--foreground)))`,+ }}+ />+ );+}++/** Topic chip colored by the topic's slot in the published map. */+function TopicChip({ name, mapIndex }: { name: string; mapIndex: number }) {+ return (+ <span className="flex min-w-0">+ <BadgeShell color={treemapBadgeColor(mapIndex)}>+ <TopicDot rank={mapIndex} />+ <span className="truncate" title={name}>+ {name}+ </span>+ </BadgeShell>+ </span>+ );+}++/** Trace header trigger: the trace's first topic and how many more it has. */+function TraceTopicsTrigger({+ facets,+ triggerProps,+}: {+ facets: TraceTopicFacet[];+ triggerProps: Record<string, unknown>;+}) {+ const assigned = facets.filter((facet) => facet.topicName);+ const lead = assigned[0];+ return (+ <span+ className="ph-no-capture hover:bg-muted inline-flex min-w-0 cursor-pointer rounded-sm px-2"+ {...triggerProps}+ >+ {lead?.topicName ? (+ <BadgeShell color="ghost" font="mono">+ <TopicDot rank={lead.topicIndex} />+ <span className="truncate" title={lead.topicName}>+ {lead.topicName}+ </span>+ {assigned.length > 1 && (+ <span className="text-muted-foreground shrink-0">+ +{assigned.length - 1}+ </span>+ )}+ </BadgeShell>+ ) : (+ <Badge color="ghost" font="mono" text="Topics" title="" />+ )}+ </span>+ );+}++/** Card listing every facet result of a trace; the caller renders the trigger. */+export function TraceTopicsHoverCard({+ projectId,+ facets,+ children,+}: {+ projectId: string;+ facets: TraceTopicFacet[];+ children: (triggerProps: Record<string, unknown>) => ReactNode;+}) {+ return (+ <HoverCard+ openDelay={200}+ placement="bottom-end"+ content={+ <div className="ph-no-capture flex max-h-[70vh] w-120 flex-col overflow-y-auto px-4">+ <div className="flex items-center justify-between gap-2 border-b py-2.5">+ <span className="text-sm font-bold">Topics</span>+ <TextLink+ path={`/project/${encodeURIComponent(projectId)}/topics`}+ value="Open in Topics →"+ />+ </div>+ {facets.map((facet) => (+ <div+ key={facet.facetId}+ className="flex flex-col gap-1.5 border-b py-3 last:border-b-0"+ >+ <div className="flex items-center justify-between gap-3">+ <span className="text-muted-foreground shrink-0 text-sm">+ {facet.name}+ </span>+ {facet.topicName && (+ <TopicChip+ name={facet.topicName}+ mapIndex={facet.topicIndex}+ />+ )}+ {!facet.summary && (+ <span className="text-muted-foreground text-sm">+ {facet.outcome+ ? outcomeLabel(facet.outcome)+ : "Not analyzed yet"}+ </span>+ )}+ </div>+ {facet.summary && (+ <p className="text-sm text-pretty">{facet.summary}</p>+ )}+ </div>+ ))}+ </div>+ }+ >+ {({ getTriggerProps }) => children(getTriggerProps())}+ </HoverCard>+ );+}++TraceTopicsHoverCard.Trigger = TraceTopicsTrigger;diff --git a/web/src/features/traces/components/TraceHeader.tsx b/web/src/features/traces/components/TraceHeader.tsxindex 0afee4cbad..729933ad5f 100644--- a/web/src/features/traces/components/TraceHeader.tsx+++ b/web/src/features/traces/components/TraceHeader.tsx@@ -24,6 +24,10 @@ import { useTraceData } from "@/src/features/traces/contexts/TraceDataProvider";import { useViewPreferences } from "@/src/features/traces/contexts/ViewPreferencesProvider";import { aggregateTraceMetrics } from "@/src/features/traces/fns/traceAggregation";import { cn } from "@/src/utils/tailwind";+import {+ TraceTopicsHoverCard,+ useTraceTopicFacets,+} from "@/src/features/topics/trace/TraceTopicsHoverCard";const MAX_VISIBLE_TAGS = 3;@@ -68,6 +72,7 @@ function TraceHeaderLoading() {function LoadedTraceHeader() {const { trace, observations, mergedScores } = useTraceData();const [showAllTags, setShowAllTags] = useState(false);+ const topicFacets = useTraceTopicFacets(trace);const handleShowFewerTags = () => {setShowAllTags(false);};@@ -118,6 +123,19 @@ function LoadedTraceHeader() {{trace.userId && (<UserIdBadge userId={trace.userId} projectId={trace.projectId} />)}+ {topicFacets.some((facet) => facet.summary) && (+ <TraceTopicsHoverCard+ projectId={trace.projectId}+ facets={topicFacets}+ >+ {(triggerProps) => (+ <TraceTopicsHoverCard.Trigger+ facets={topicFacets}+ triggerProps={triggerProps}+ />+ )}+ </TraceTopicsHoverCard>+ )}{(traceScores.length > 0 || trace.tags.length > 0) && (<div className="flex min-w-0 flex-wrap items-center gap-1">{traceScores.length > 0 && (diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.tsindex c550326a09..420a80f5d0 100644--- a/worker/src/features/topics/storage.integration.test.ts+++ b/worker/src/features/topics/storage.integration.test.ts@@ -17,6 +17,7 @@ import {getTopicClusteringSummaryStats,getTopicSummaryCounts,getTopicExecutionCosts,+ getTopicTraceUnitTimeRange,createTopicFacet,createTopicFacetVersion,ensureDefaultTopicFacets,@@ -654,6 +655,41 @@ describe("Topics definition persistence", () => {processedAt: newerVersion.processedAt,},]);+ expect(+ await Promise.all(+ [[summary.traceId], ["other-trace"]].map((traceIds) =>+ getLatestFacetSummaries(+ projectId,+ "facet",+ undefined,+ oldRange,+ traceIds,+ ),+ ),+ ),+ ).toEqual([+ [+ {+ ...displaySummary,+ facetVersion: newerVersion.facetVersion,+ processedAt: newerVersion.processedAt,+ },+ ],+ [],+ ]);+ expect(+ await Promise.all(+ [summary.traceId!, "other-trace"].map((traceId) =>+ getTopicTraceUnitTimeRange(projectId, traceId, oldRange),+ ),+ ),+ ).toEqual([+ {+ from: new Date(summary.unitStartTime),+ to: new Date(Date.parse(summary.unitStartTime) + 1),+ },+ null,+ ]);expect(await getLatestFacetSummaries(projectId, "facet", undefined, newRange),).toEqual([]);