🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(topics): summarize all facets of a trace in one call (#18013)
Lysander Kieselbe0b074e58c5
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex ecb5d37141..09b27886f8 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -126,8 +126,11 @@ Existing topic names remain until their definitions change during **Update topic## Trace-batch connectionAfter the trace-batch job assembles a transcript, it summarizes that same-transcript for each current facet of an allowlisted project. It does not load-the trace from ClickHouse again. Projects outside+transcript for all current facets of an allowlisted project in one model call:+the system prompt lists every facet under a key, asks for each to be answered+independently with short evidence notes, and returns one entry per facet. The+call's usage is recorded on the first facet's row. It does not load the trace+from ClickHouse again. Projects outside`LANGFUSE_TOPICS_ENABLED_PROJECT_IDS` are skipped. A finished facet version isskipped on retry. A failing trace does not stop the batch: every trace isprocessed and outcomes are counted once per job (`langfuse.topics.trace_outcomes`diff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.tsindex fc7024c903..bb2e73be55 100644--- a/worker/src/features/topics/models.test.ts+++ b/worker/src/features/topics/models.test.ts@@ -26,6 +26,7 @@ import {embedTopicSummary,nameTopicGroup,summarizeTopicTrace,+ summarizeTopicTraceFacets,} from "./models";import { topicProcessingConfigSchema } from "@langfuse/shared/topics";@@ -161,6 +162,65 @@ describe("Topics naming boundary", () => {expect(state.call).not.toHaveBeenCalled();});+ it("summarizes all facets of a trace in one request and checks its size first", async () => {+ state.call.mockResolvedValue({+ output: {+ intent_1: {+ notes: "n",+ summary: "Export sales.",+ status: "applicable",+ },+ issues_2: { notes: "n", summary: "", status: "not_applicable" },+ },+ usage: { inputTokens: 100, outputTokens: 30 },+ });+ const config = topicProcessingConfigSchema.parse({+ summaryModel: "us.openai.gpt-5.6-luna",+ });+ const issues = {+ ...facet,+ facetId: "issues",+ prompt: "Describe the main problem.",+ };+ const facets = [+ { key: "intent_1", facet },+ { key: "issues_2", facet: issues },+ ];+ await summarizeTopicTraceFacets(facets, "RAW_TRANSCRIPT_SENTINEL", config);+ expect(state.call).toHaveBeenCalledOnce();+ const request = state.call.mock.calls[0][0];+ expect(request.messages[0].content).toContain(+ `<facet key="intent_1">\n${facet.prompt}\n</facet>`,+ );+ expect(request.messages[0].content).toContain(+ `<facet key="issues_2">\n${issues.prompt}\n</facet>`,+ );+ expect(request.maxOutputTokens).toBe(config.maxOutputTokens * 2);+ // Every facet key is required in the structured output.+ expect(+ request.schema.safeParse({+ intent_1: {+ notes: "n",+ summary: "Export sales.",+ status: "applicable",+ },+ }).success,+ ).toBe(false);++ state.call.mockClear();+ await expect(+ summarizeTopicTraceFacets(+ facets,+ "Trace evidence. ".repeat(1000),+ topicProcessingConfigSchema.parse({+ maxInputTokens: 256,+ summaryModel: "us.openai.gpt-5.6-luna",+ }),+ ),+ ).rejects.toThrow(/above this run's 256-token input limit/);+ expect(state.call).not.toHaveBeenCalled();+ });+it("names the complete cohort without shortening member summaries", async () => {const members = Array.from({ length: 400 }, (_, index) => ({id: String(index).padStart(48, "0"),diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.tsindex c6721c312e..05d031e5d3 100644--- a/worker/src/features/topics/models.ts+++ b/worker/src/features/topics/models.ts@@ -216,6 +216,62 @@ export async function summarizeTopicTrace();}+/**+ * Summarizes all given facets of one trace in a single call. The system prompt+ * is identical across traces; each facet is listed under its key and answered+ * independently, with short evidence notes before each summary.+ */+export async function summarizeTopicTraceFacets(+ facets: { key: string; facet: TopicFacetVersion }[],+ text: string,+ config: TopicProcessingConfig,+) {+ const models = requireTopicsModelConfig();+ const model = config.summaryModel;+ if (!model || model !== models.summaryModel)+ throw new TopicsProviderUnavailable(+ "Configure the matching LANGFUSE_TOPICS_SUMMARY_MODEL on web and worker before running Topics.",+ "authentication",+ );+ const entry = z.object({+ notes: z+ .string()+ .describe(+ "At most 25 words: the transcript evidence this facet rests on. Not shown to anyone.",+ ),+ ...summarySchema.shape,+ });+ const schema = z.object(+ Object.fromEntries(facets.map(({ key }) => [key, entry])),+ );+ // Without these rules, a whole-run facet such as Intent narrows to the end of the run.+ const system = `${SUMMARY_SYSTEM_PROMPT}++This request covers ${facets.length} facets of the same run. Treat each facet as a separate task:+- For each facet, read the whole transcript again for what that facet asks about. Facets are independent: what you write for one facet must not narrow or shape another.+- A facet about the run as a whole covers all of it, from the first request to the last turn, even when other facets concentrate on how the run ended.+- Follow each facet's own format and apply the status rules to each facet separately.+For each facet, first write brief notes on the evidence it rests on, then its summary and status. Return one entry per facet key.++${facets.map(({ key, facet }) => `<facet key="${key}">\n${facet.prompt}\n</facet>`).join("\n\n")}`;+ const input = `<transcript>\n${text}\n</transcript>\n\nWrite the summaries now, one per facet key, each in its facet's format.`;+ const countedInputTokens =+ countTopicRequestTokens(system, input, schema) + 256;+ if (countedInputTokens > config.maxInputTokens)+ throw new Error(+ `The shared trace transcript and instructions are ${countedInputTokens} tokens, above this run's ${config.maxInputTokens}-token input limit. No model call was made; the transcript is never shortened per facet.`,+ );+ return structuredCall(+ system,+ input,+ schema,+ config.maxInputTokens,+ config.maxOutputTokens * facets.length,+ "summary",+ model,+ );+}+export async function nameTopicGroup(group: {members: { id: string; summary: string }[];contrasts: { id: string; summary: string }[];diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.tsindex 8087ea1dd0..dcdf86d9d5 100644--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts@@ -23,7 +23,7 @@ vi.mock("@langfuse/shared/topics/server", () => ({TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v2",}));vi.mock("./models", () => ({- summarizeTopicTrace: (...args: unknown[]) => state.summarize(...args),+ summarizeTopicTraceFacets: (...args: unknown[]) => state.summarize(...args),embedTopicSummary: (...args: unknown[]) => state.embed(...args),}));@@ -91,7 +91,9 @@ beforeEach(() => {state.stored.mockResolvedValue([]);state.write.mockResolvedValue(undefined);state.summarize.mockResolvedValue({- output: { summary: "Export monthly sales.", status: "applicable" },+ output: {+ intent_1: { summary: "Export monthly sales.", status: "applicable" },+ },...usage,});state.embed.mockResolvedValue({@@ -131,7 +133,9 @@ describe("summarizeAssembledTrace", () => {expect(state.facets).toHaveBeenCalledWith("project-a");expect(state.summarize).toHaveBeenCalledTimes(1);const submitted = state.summarize.mock.calls[0];- expect(submitted[0].prompt).toBe(facet.versions[0].prompt);+ expect(submitted[0]).toEqual([+ { key: "intent_1", facet: facet.versions[0] },+ ]);expect(submitted[1]).toContain("Export monthly sales");expect(submitted[1]).not.toContain("observation");const written = state.write.mock.calls[0][0][0];@@ -192,6 +196,75 @@ describe("summarizeAssembledTrace", () => {);});+ it("summarizes all pending facets in one call and records its usage once", async () => {+ const issues: TopicFacet = {+ ...facet,+ id: "facet-2",+ name: "Issues",+ versions: [{ ...facet.versions[0], facetId: "facet-2", version: 1 }],+ };+ state.facets.mockResolvedValue([facet, issues]);+ state.summarize.mockResolvedValue({+ output: {+ intent_1: { summary: "Export monthly sales.", status: "applicable" },+ issues_2: { summary: "", status: "not_applicable" },+ },+ ...usage,+ });+ await summarizeAssembledTrace({+ projectId: "project-a",+ traceId: "trace-1",+ traceTimestamp: "2026-09-22T12:00:00.000Z",+ environment: "default",+ traceName: "agent-turn",+ transcript,+ });+ expect(state.summarize).toHaveBeenCalledOnce();+ const [intent, issue] = state.write.mock.calls[0][0];+ expect(intent).toMatchObject({ facetId: "facet-1", state: "complete" });+ expect(intent.usageDetails.summary_input).toBe(20);+ expect(issue).toMatchObject({+ facetId: "facet-2",+ state: "not_applicable",+ usageDetails: {},+ });+ });++ it("keeps the other facets when one facet's output is invalid", async () => {+ const issues: TopicFacet = {+ ...facet,+ id: "facet-2",+ name: "Issues",+ versions: [{ ...facet.versions[0], facetId: "facet-2", version: 1 }],+ };+ state.facets.mockResolvedValue([facet, issues]);+ state.summarize.mockResolvedValue({+ output: {+ intent_1: { summary: "", status: "applicable" },+ issues_2: { summary: "Retried a failed export.", status: "applicable" },+ },+ ...usage,+ });+ await expect(+ summarizeAssembledTrace({+ projectId: "project-a",+ traceId: "trace-1",+ traceTimestamp: "2026-09-22T12:00:00.000Z",+ environment: "default",+ traceName: "agent-turn",+ transcript,+ }),+ ).rejects.toThrow(+ "Applicable facet summary must contain a concise summary.",+ );+ const written = state.write.mock.calls[0][0];+ expect(written).toHaveLength(1);+ expect(written[0]).toMatchObject({ facetId: "facet-2", state: "complete" });+ // The shared call's usage moves to the first row that is written.+ expect(written[0].usageDetails.summary_input).toBe(20);+ expect(written[0]).not.toHaveProperty("output");+ });+it("propagates a provider failure for the trace outcome", async () => {state.summarize.mockRejectedValue(new Error("provider unavailable"));await expect(diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.tsindex 74050b27c1..16103a61e9 100644--- a/worker/src/features/topics/summarizeAssembledTrace.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.ts@@ -13,14 +13,30 @@ import {} from "@langfuse/shared/topics";import { recordIncrement, type Transcript } from "@langfuse/shared/src/server";import { prepareAssembledTopicTranscript } from "./assembledTranscript";-import { embedTopicSummary, summarizeTopicTrace } from "./models";+import { embedTopicSummary, summarizeTopicTraceFacets } from "./models";import { TopicsProviderUnavailable } from "./provider-error";-import { mergeTopicModelUsage, topicSummaryOutputError } from "./summaryResult";+import {+ mergeTopicModelUsage,+ topicSummaryOutputError,+ type TopicModelUsage,+} from "./summaryResult";import { getTopicsModelConfig } from "@langfuse/shared/topics/server";+type FacetOutput = {+ summary: string;+ status: "applicable" | "not_applicable" | "insufficient_input";+};+const NO_USAGE: TopicModelUsage = {+ providedUsageDetails: {},+ usageDetails: {},+ providedCostDetails: {},+ costDetails: {},+};+/**- * Summarizes one trace from the transcript the batch job already assembled.- * Does not load observations again.+ * Summarizes one trace from the transcript the batch job already assembled,+ * with one model call for all of its pending facets. Does not load+ * observations again.*/export async function summarizeAssembledTrace(input: {projectId: string;@@ -31,82 +47,109 @@ export async function summarizeAssembledTrace(input: {transcript: Transcript | null;}): Promise<"disabled" | "unchanged" | "summarized"> {if (!isTopicsProjectEnabled(input.projectId)) return "disabled";- let prepared: ReturnType<typeof prepareAssembledTopicTranscript> | undefined;const models = getTopicsModelConfig();if (!models.summaryModel || !models.embeddingModel)throw new TopicsProviderUnavailable("Configure LANGFUSE_TOPICS_SUMMARY_MODEL and LANGFUSE_TOPICS_EMBEDDING_MODEL before processing Topics traces.","authentication",);+ const { embeddingModel } = models;const facets = await ensureDefaultTopicFacets(input.projectId);- const versions = facets.flatMap((facet) =>- facet.projectId === input.projectId ? facet.versions.slice(0, 1) : [],- );const config = topicProcessingConfigSchema.parse({summaryModel: models.summaryModel,});const embeddingConfig = topicEmbeddingConfigSchema.parse({- embeddingModel: models.embeddingModel,+ embeddingModel,});- const dimensions = embeddingConfig.embeddingDimensions;const timestamp = Date.parse(input.traceTimestamp);const timeRange = {from: new Date(timestamp),to: new Date(timestamp + 1),};- let written = 0;- for (const facet of versions) {+ const pending: { name: string; version: TopicFacetVersion }[] = [];+ for (const facet of facets) {+ const version = facet.versions[0];+ if (facet.projectId !== input.projectId || !version) continue;const stored = await listTopicSummaries(input.projectId,{traceIds: [input.traceId],- facetId: facet.facetId,- facetVersion: facet.version,+ facetId: version.facetId,+ facetVersion: version.version,},timeRange,);- if (stored.length) continue;- if (!prepared) prepared = prepareAssembledTopicTranscript(input.transcript);- const summary = await summarizeFacet({- ...input,- facet,- text: prepared.text,- hasContent: prepared.hasContent,+ if (!stored.length) pending.push({ name: facet.name, version });+ }+ if (!pending.length) return "unchanged";++ const prepared = prepareAssembledTopicTranscript(input.transcript);+ const bases = pending.map(({ version }) =>+ baseSummary(input, version, config, embeddingModel),+ );+ let rows = bases;+ let failure: unknown;+ if (prepared.hasContent) {+ const keyed = pending.map(({ name, version }, index) => ({+ key: `${name.toLowerCase().replace(/[^a-z0-9]+/g, "_")}_${index + 1}`,+ facet: version,+ }));+ const result = await summarizeTopicTraceFacets(+ keyed,+ prepared.text,config,- dimensions,- embeddingModel: embeddingConfig.embeddingModel,- });- await writeTopicSummaries([summary]);- recordIncrement("langfuse.topics.facet_summaries", 1, {- state: summary.state,- });- written++;+ );+ const outputs = result.output as Record<string, FacetOutput>;+ const usage: TopicModelUsage = {+ providedUsageDetails: result.providedUsageDetails,+ usageDetails: result.usageDetails,+ providedCostDetails: result.providedCostDetails,+ costDetails: result.costDetails,+ };+ rows = [];+ // A failed facet (invalid output, embedding error) must not discard the others.+ for (const [index, { key }] of keyed.entries()) {+ try {+ rows.push(+ await finalizeSummary(+ bases[index],+ outputs[key],+ // One call serves every facet of the trace; record its usage once.+ rows.length ? NO_USAGE : usage,+ embeddingConfig.embeddingDimensions,+ embeddingModel,+ ),+ );+ } catch (error) {+ if (failure === undefined) failure = error;+ }+ }}- return written ? "summarized" : "unchanged";+ if (rows.length) await writeTopicSummaries(rows);+ for (const row of rows)+ recordIncrement("langfuse.topics.facet_summaries", 1, { state: row.state });+ if (failure) throw failure;+ return "summarized";}-async function summarizeFacet(input: {- projectId: string;- traceId: string;- traceTimestamp: string;- environment: string;- traceName: string;- facet: TopicFacetVersion;- text: string;- hasContent: boolean;- config: ReturnType<typeof topicProcessingConfigSchema.parse>;- dimensions: number;- embeddingModel?: string;-}): Promise<TopicSummary> {- const source = {+function baseSummary(+ input: {+ projectId: string;+ traceId: string;+ traceTimestamp: string;+ environment: string;+ traceName: string;+ },+ facet: TopicFacetVersion,+ config: ReturnType<typeof topicProcessingConfigSchema.parse>,+ embeddingModel: string,+): TopicSummary {+ return {projectId: input.projectId,- facetId: input.facet.facetId,- facetVersion: input.facet.version,+ facetId: facet.facetId,+ facetVersion: facet.version,traceId: input.traceId,sessionId: null,- };- const base: TopicSummary = {- ...source,triggerType: "manual_poc",unitStartTime: input.traceTimestamp,environment: input.environment,@@ -116,48 +159,44 @@ async function summarizeFacet(input: {embedding: [],transcriptId: "trace-batch",transcriptVersion: TOPICS_TRANSCRIPT_VERSION,- summaryModel: input.config.summaryModel!,- embeddingModel: input.embeddingModel!,- providedUsageDetails: {},- usageDetails: {},- providedCostDetails: {},- costDetails: {},+ summaryModel: config.summaryModel!,+ embeddingModel,+ ...NO_USAGE,processedAt: new Date().toISOString(),metadata: { input: "assembled-transcript" },};- if (!input.hasContent) return base;- const result = await summarizeTopicTrace(- input.facet,- input.text,- input.config,- );- const applicable = result.output.status === "applicable";- const summary = result.output.summary.trim();- const error = topicSummaryOutputError(result.output);+}++async function finalizeSummary(+ base: TopicSummary,+ output: FacetOutput | undefined,+ usage: TopicModelUsage,+ dimensions: number,+ embeddingModel: string,+): Promise<TopicSummary> {+ if (!output)+ throw new TopicsProviderUnavailable(+ "The model response is missing a facet.",+ "invalid_output",+ );+ const error = topicSummaryOutputError(output);if (error) throw new TopicsProviderUnavailable(error, "invalid_output");- if (!applicable) {+ if (output.status !== "applicable")return {...base,state:- result.output.status === "not_applicable"+ output.status === "not_applicable"? "not_applicable": "insufficient_input",- providedUsageDetails: result.providedUsageDetails,- usageDetails: result.usageDetails,- providedCostDetails: result.providedCostDetails,- costDetails: result.costDetails,+ ...usage,};- }- const embedded = await embedTopicSummary(- summary,- input.dimensions,- input.embeddingModel!,- );+ const summary = output.summary.trim();+ const embedded = await embedTopicSummary(summary, dimensions, embeddingModel);return {...base,state: "complete",summary,embedding: embedded.embedding,- ...mergeTopicModelUsage(result, embedded),+ ...mergeTopicModelUsage(usage, embedded),};}