🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(topics): write facet summaries once per trace batch (#18717)
Lysander Kiesel147d7ef1c974
diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.tsindex c589c9b120..dfdddc384c 100644--- a/packages/shared/src/server/topics/clickhouse.ts+++ b/packages/shared/src/server/topics/clickhouse.ts@@ -519,18 +519,21 @@ export async function getTopicSummaryCounts(}));}+/** writeTopicSummaries rejects the whole insert if any row fails this check. */+export function isValidTopicSummary(row: TopicSummary): boolean {+ return (+ topicSourceSchema.safeParse(row).success &&+ typeof row.environment === "string" &&+ row.embedding.every(Number.isFinite) &&+ !(row.state === "complete" && row.embedding.length === 0) &&+ row.state !== "summarized"+ );+}+export async function writeTopicSummaries(rows: TopicSummary[]): Promise<void> {if (!rows.length) return;- for (const row of rows) {- if (- !topicSourceSchema.safeParse(row).success ||- typeof row.environment !== "string" ||- !row.embedding.every(Number.isFinite) ||- (row.state === "complete" && !row.embedding.length) ||- row.state === "summarized"- )- throw new Error("Invalid Topics summary result.");- }+ if (!rows.every(isValidTopicSummary))+ throw new Error("Invalid Topics summary result.");await insertTopicRows("topic_facet_summaries", rows, (row) => ({project_id: row.projectId,facet_id: row.facetId,diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.tsindex cf4c28ae12..f158fa2325 100644--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts@@ -4,13 +4,12 @@ import {recordIncrement,type Transcript,} from "@langfuse/shared/src/server";-import type { TopicFacet } from "@langfuse/shared/topics";+import type { TopicFacet, TopicSummary } from "@langfuse/shared/topics";import type { TopicsModels } from "@langfuse/shared/topics/server";const state = vi.hoisted(() => ({facets: vi.fn(),stored: vi.fn(),- write: vi.fn(),summarize: vi.fn(),embed: vi.fn(),setAttribute: vi.fn(),@@ -18,7 +17,6 @@ const state = vi.hoisted(() => ({vi.mock("@langfuse/shared/topics/server", () => ({listTopicSummaries: (...args: unknown[]) => state.stored(...args),- writeTopicSummaries: (...args: unknown[]) => state.write(...args),TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v3",}));vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({@@ -108,18 +106,28 @@ const usage = {},};+let summaries: TopicSummary[] = [];+// Keeps the returned rows for assertions and rethrows a partial failure, as the batch job reports it.+const summarize = async (+ input: Parameters<typeof summarizeAssembledTrace>[0],+) => {+ const result = await summarizeAssembledTrace(input);+ summaries = result.rows;+ if (result.failure !== undefined) throw result.failure;+ return result.outcome;+};+beforeEach(() => {vi.mocked(recordDistribution).mockClear();vi.mocked(recordIncrement).mockClear();state.facets.mockReset();state.stored.mockReset();- state.write.mockReset();+ summaries = [];state.summarize.mockReset();state.embed.mockReset();state.setAttribute.mockClear();state.facets.mockReturnValue([facet]);state.stored.mockResolvedValue([]);- state.write.mockResolvedValue(undefined);state.summarize.mockResolvedValue({output: {intent_1: { summary: "Export monthly sales.", status: "applicable" },@@ -137,7 +145,7 @@ beforeEach(() => {describe("summarizeAssembledTrace", () => {it("skips internal traces before reading summaries, calling models or writing", async () => {- const result = await summarizeAssembledTrace({+ const result = await summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -152,11 +160,11 @@ describe("summarizeAssembledTrace", () => {expect(state.stored).not.toHaveBeenCalled();expect(state.summarize).not.toHaveBeenCalled();expect(state.embed).not.toHaveBeenCalled();- expect(state.write).not.toHaveBeenCalled();+ expect(summaries).toEqual([]);});it("skips a finished job retry but summarizes a later redispatch", async () => {- await summarizeAssembledTrace({+ await summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -179,7 +187,7 @@ describe("summarizeAssembledTrace", () => {source_trace_id: "trace-1",source_environment: "default",});- const written = state.write.mock.calls[0][0][0];+ const written = summaries[0];expect(written).toMatchObject({projectId: "project-a",traceId: "trace-1",@@ -214,8 +222,8 @@ describe("summarizeAssembledTrace", () => {{ ...written, processedAt: "2026-09-22T12:02:00.000Z" },]);state.summarize.mockClear();- state.write.mockClear();- await summarizeAssembledTrace({+ summaries = [];+ await summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -227,8 +235,8 @@ describe("summarizeAssembledTrace", () => {facets: state.facets(),});expect(state.summarize).not.toHaveBeenCalled();- expect(state.write).not.toHaveBeenCalled();- await summarizeAssembledTrace({+ expect(summaries).toEqual([]);+ await summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -240,10 +248,8 @@ describe("summarizeAssembledTrace", () => {facets: state.facets(),});expect(state.summarize).toHaveBeenCalledOnce();- expect(state.write).toHaveBeenCalledOnce();- expect(state.write.mock.calls[0][0][0].unitStartTime).toBe(- written.unitStartTime,- );+ expect(summaries).toHaveLength(1);+ expect(summaries[0].unitStartTime).toBe(written.unitStartTime);expect(state.stored).toHaveBeenCalledTimes(3);});@@ -269,7 +275,7 @@ describe("summarizeAssembledTrace", () => {},...usage,});- await summarizeAssembledTrace({+ await summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -281,7 +287,7 @@ describe("summarizeAssembledTrace", () => {facets: state.facets(),});expect(state.summarize).toHaveBeenCalledOnce();- const [intent, issue] = state.write.mock.calls[0][0];+ const [intent, issue] = summaries;expect(intent).toMatchObject({ facetId: "facet-1", state: "complete" });expect(intent.usageDetails.summary_input).toBe(20);expect(issue).toMatchObject({@@ -298,7 +304,7 @@ describe("summarizeAssembledTrace", () => {},...usage,});- await summarizeAssembledTrace({+ await summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -309,7 +315,7 @@ describe("summarizeAssembledTrace", () => {models,facets: state.facets(),});- const [written] = state.write.mock.calls[0][0];+ const [written] = summaries;expect(written).toMatchObject({ state: "not_applicable" });expect(written.summary ?? "").toBe("");expect(state.embed).not.toHaveBeenCalled();@@ -331,7 +337,7 @@ describe("summarizeAssembledTrace", () => {...usage,});await expect(- summarizeAssembledTrace({+ summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -345,7 +351,7 @@ describe("summarizeAssembledTrace", () => {).rejects.toThrow("Applicable facet summary must contain a concise summary.",);- const written = state.write.mock.calls[0][0];+ const written = summaries;expect(written).toHaveLength(1);expect(written[0]).toMatchObject({ facetId: "facet-2", state: "complete" });// The shared call's usage moves to the first row that is written.@@ -389,7 +395,7 @@ describe("summarizeAssembledTrace", () => {},...usage,});- await summarizeAssembledTrace({+ await summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -426,7 +432,7 @@ describe("summarizeAssembledTrace", () => {const error = new Error("embedding provider unavailable");state.embed.mockRejectedValueOnce(error);await expect(- summarizeAssembledTrace({+ summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -439,8 +445,7 @@ describe("summarizeAssembledTrace", () => {}),).rejects.toBe(error);expect(state.summarize).toHaveBeenCalledOnce();- expect(state.write).toHaveBeenCalledOnce();- expect(state.write.mock.calls[0][0]).toEqual([+ expect(summaries).toEqual([expect.objectContaining({facetId: "facet-2",state: "not_applicable",@@ -469,7 +474,7 @@ describe("summarizeAssembledTrace", () => {const error = new Error("provider unavailable");state.summarize.mockRejectedValue(error);await expect(- summarizeAssembledTrace({+ summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -485,7 +490,7 @@ describe("summarizeAssembledTrace", () => {"langfuse.topics.trace_outcome","failed",);- expect(state.write).not.toHaveBeenCalled();+ expect(summaries).toEqual([]);expect(recordDistribution).toHaveBeenCalledWith("langfuse.topics.stage_duration_ms",expect.any(Number),@@ -507,7 +512,7 @@ describe("summarizeAssembledTrace", () => {...usage,});await expect(- summarizeAssembledTrace({+ summarize({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",@@ -520,7 +525,7 @@ describe("summarizeAssembledTrace", () => {}),).rejects.toMatchObject({ reason: "invalid_output" });expect(state.embed).not.toHaveBeenCalled();- expect(state.write).not.toHaveBeenCalled();+ expect(summaries).toEqual([]);expect(recordDistribution).toHaveBeenCalledWith("langfuse.topics.stage_duration_ms",expect.any(Number),diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.tsindex 3928f8d184..5010b1eb1d 100644--- a/worker/src/features/topics/summarizeAssembledTrace.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.ts@@ -1,7 +1,6 @@import {listTopicSummaries,TOPICS_TRANSCRIPT_VERSION,- writeTopicSummaries,type TopicsModels,} from "@langfuse/shared/topics/server";import {@@ -13,7 +12,6 @@ import {import {instrumentAsync,isInternalEvalEnvironment,- recordIncrement,type Transcript,} from "@langfuse/shared/src/server";import { prepareAssembledTopicTranscript } from "./assembledTranscript";@@ -49,15 +47,23 @@ type AssembledTraceInput = {facets: TopicFacet[];};+type AssembledTraceResult = {+ outcome: "disabled" | "unchanged" | "summarized";+ rows: TopicSummary[];+ // Set when some facets failed; rows still holds the valid ones.+ failure?: unknown;+};+/*** Summarizes one trace from the transcript the batch job already assembled,* with one model call for all of its configured facets. Does not load- * observations again.+ * observations again or write; the batch job stores the returned rows.*/export async function summarizeAssembledTrace(input: AssembledTraceInput,-): Promise<"disabled" | "unchanged" | "summarized"> {- if (isInternalEvalEnvironment(input.environment)) return "disabled";+): Promise<AssembledTraceResult> {+ if (isInternalEvalEnvironment(input.environment))+ return { outcome: "disabled", rows: [] };return instrumentAsync({ name: "topics-trace-summary", traceScope: "topics" },async (span) => {@@ -66,14 +72,18 @@ export async function summarizeAssembledTrace("langfuse.trace.id": input.traceId,});const metrics = new TopicMetrics();+ let result: AssembledTraceResult | undefined;try {- const outcome = await metrics.measure("trace", () =>- summarizeEnabledTrace(input, metrics),- );- span.setAttribute("langfuse.topics.trace_outcome", outcome);- return outcome;+ await metrics.measure("trace", async () => {+ result = await summarizeEnabledTrace(input, metrics);+ if (result.failure !== undefined) throw result.failure;+ });+ span.setAttribute("langfuse.topics.trace_outcome", result!.outcome);+ return result!;} catch (error) {span.setAttribute("langfuse.topics.trace_outcome", "failed");+ // A partial failure still returns the valid facets' rows.+ if (result && result.failure === error) return result;throw error;}},@@ -83,7 +93,7 @@ export async function summarizeAssembledTrace(async function summarizeEnabledTrace(input: AssembledTraceInput,metrics: TopicMetrics,-): Promise<"unchanged" | "summarized"> {+): Promise<AssembledTraceResult> {const { models, facets } = input;const embeddingModel = models.embedding.model;const config = topicProcessingConfigSchema.parse({@@ -125,7 +135,7 @@ async function summarizeEnabledTrace(continue;pending.push({ name: facet.name, version, builtIn: facet.isBuiltIn });}- if (!pending.length) return "unchanged";+ if (pending.length === 0) return { outcome: "unchanged", rows: [] };const prepared = prepareAssembledTopicTranscript(input.transcript);const bases = pending.map(({ version }) =>@@ -232,12 +242,7 @@ async function summarizeEnabledTrace(});}}- if (rows.length)- await metrics.measure("storage", () => writeTopicSummaries(rows));- for (const row of rows)- recordIncrement("langfuse.topics.facet_summaries", 1, { state: row.state });- if (failure) throw failure;- return "summarized";+ return { outcome: "summarized", rows, failure };}function baseSummary(diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex ec8605a8a9..13b3dff61f 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -34,10 +34,16 @@ import {getEnabledTopicsModels,listTopicFacets,listTopicRules,+ isValidTopicSummary,pauseTopicsModels,+ writeTopicSummaries,type TopicsModels,} from "@langfuse/shared/topics/server";-import type { TopicFacet, TopicRule } from "@langfuse/shared/topics";+import type {+ TopicFacet,+ TopicRule,+ TopicSummary,+} from "@langfuse/shared/topics";import { recordTraceBatchTranscript } from "../../features/traceBatching/traceBatchTranscript";import * as traceBatchTranscript from "../../features/traceBatching/traceBatchTranscript";import {@@ -55,7 +61,10 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({}));vi.mock("../../features/topics/summarizeAssembledTrace", () => ({- summarizeAssembledTrace: vi.fn(async () => "unchanged"),+ summarizeAssembledTrace: vi.fn(async () => ({+ outcome: "unchanged",+ rows: [],+ })),}));vi.mock("@langfuse/shared/topics/server", () => ({@@ -67,6 +76,8 @@ vi.mock("@langfuse/shared/topics/server", () => ({listTopicFacets: vi.fn(async () => []),listTopicRules: vi.fn(async () => []),pauseTopicsModels: vi.fn(),+ writeTopicSummaries: vi.fn(),+ isValidTopicSummary: vi.fn(() => true),}));// Exercise real tokenization without starting the compiled worker-thread pool.@@ -606,11 +617,11 @@ describe("trace batch queue", () => {vi.mocked(summarizeAssembledTrace).mockImplementationOnce(async () => {await first;- return "summarized";+ return { outcome: "summarized", rows: [] };}).mockImplementationOnce(async () => {await second;- return "summarized";+ return { outcome: "summarized", rows: [] };});vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {for (const traceId of ["a", "b", "c"]) {@@ -684,10 +695,13 @@ describe("trace batch queue", () => {resolveSecond();await processing;env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = originalCap;- vi.mocked(summarizeAssembledTrace).mockResolvedValue("unchanged");+ vi.mocked(summarizeAssembledTrace).mockResolvedValue({+ outcome: "unchanged",+ rows: [],+ });}});- it("finishes the batch past a failed trace, reports outcomes and does not re-enqueue the failure", async () => {+ it("finishes the batch past a failed trace, writes rows once, retries transient errors and drops only failing rows", async () => {const traces = ["a", "b", "c"];vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {for (const projectId of traces)@@ -712,11 +726,19 @@ describe("trace batch queue", () => {};});vi.mocked(summarizeAssembledTrace).mockImplementation(- async ({ projectId }) => {- if (projectId === "b")- throw new TopicsProviderUnavailable("bad output", "invalid_output");- return "summarized";- },+ async ({ projectId }) => ({+ outcome: "summarized",+ rows: [{ projectId, state: "complete" } as TopicSummary],+ // A failed trace still returns its valid facets.+ ...(projectId === "b"+ ? {+ failure: new TopicsProviderUnavailable(+ "bad output",+ "invalid_output",+ ),+ }+ : {}),+ }),);const add = vi.fn();vi.spyOn(TraceBatchQueue, "getInstance").mockReturnValue({@@ -754,10 +776,132 @@ describe("trace batch queue", () => {1,{ outcome: "failed", reason: "invalid_output" },);+ expect(writeTopicSummaries).toHaveBeenCalledOnce();+ expect(+ vi+ .mocked(writeTopicSummaries)+ .mock.calls[0][0].map((row) => row.projectId),+ ).toEqual(traces);expect(add).not.toHaveBeenCalled();++ // c's invalid row is dropped before the insert; a transient error retries the rest.+ vi.mocked(recordIncrement).mockClear();+ vi.mocked(isValidTopicSummary).mockImplementation(+ (row) => row.projectId !== "c",+ );+ vi.mocked(writeTopicSummaries).mockRejectedValueOnce(+ new Error("Timeout error."),+ );+ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual(+ expect.objectContaining({ traceCount: 3 }),+ );+ const outcomeCalls = vi+ .mocked(recordIncrement)+ .mock.calls.filter(([name]) => name === "langfuse.topics.trace_outcomes");+ expect(outcomeCalls).toEqual(+ expect.arrayContaining([+ ["langfuse.topics.trace_outcomes", 1, { outcome: "summarized" }],+ [+ "langfuse.topics.trace_outcomes",+ 1,+ { outcome: "failed", reason: "invalid_output" },+ ],+ [+ "langfuse.topics.trace_outcomes",+ 1,+ { outcome: "failed", reason: "storage" },+ ],+ ]),+ );+ expect(outcomeCalls).toHaveLength(3);+ expect(writeTopicSummaries).toHaveBeenCalledTimes(3);+ expect(+ vi+ .mocked(writeTopicSummaries)+ .mock.lastCall![0].map((row) => row.projectId),+ ).toEqual(["a", "b"]);++ // Like ClickhouseWriter, an outage drops the insert once instead of splitting it.+ vi.mocked(recordIncrement).mockClear();+ vi.mocked(isValidTopicSummary).mockImplementation(() => true);+ vi.mocked(writeTopicSummaries)+ .mockReset()+ .mockRejectedValue(new Error("connect ECONNREFUSED"));+ await traceBatchQueueProcessor(job, undefined);+ expect(writeTopicSummaries).toHaveBeenCalledOnce();+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.topics.trace_outcomes",+ 2,+ { outcome: "failed", reason: "storage" },+ );+ vi.mocked(writeTopicSummaries).mockReset();+ vi.mocked(summarizeAssembledTrace).mockImplementation(async () => ({+ outcome: "unchanged",+ rows: [],+ }));+ });++ it("stores completed traces' rows before rethrowing a failed stream, so a retry skips them", async () => {+ const failure = new Error("query failed");+ const event = (projectId: string) => ({+ project_id: projectId,+ trace_id: "trace",+ environment: "default",+ span_id: "span",+ parent_span_id: null,+ start_time: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000000",+ type: "GENERATION",+ name: "generation",+ input: JSON.stringify([{ role: "user", content: projectId }]),+ output: JSON.stringify({ role: "assistant", content: "answer" }),+ metadata: {},+ level: "DEFAULT",+ status_message: null,+ tool_definitions: {},+ tool_calls: [],+ tool_call_names: [],+ });+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {+ yield event("a");+ // b's boundary completes a; b itself is cut off by the failure.+ yield event("b");+ throw failure;+ });vi.mocked(summarizeAssembledTrace).mockImplementation(- async () => "unchanged",+ async ({ projectId }) => ({+ outcome: "summarized",+ rows: [{ projectId, state: "complete" } as TopicSummary],+ }),);+ const job = {+ data: {+ id: "stream-failure",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(),+ payload: {+ traces: ["a", "b"].map((projectId) => ({+ projectId,+ traceId: "trace",+ minStart: 0,+ maxStart: 1,+ revision: "r",+ })),+ },+ },+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;++ await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(+ failure,+ );+ expect(writeTopicSummaries).toHaveBeenCalledOnce();+ expect(vi.mocked(writeTopicSummaries).mock.calls[0][0]).toEqual([+ { projectId: "a", state: "complete" },+ ]);+ vi.mocked(summarizeAssembledTrace).mockImplementation(async () => ({+ outcome: "unchanged",+ rows: [],+ }));});it("summarizes only the saved rule's facets and skips traces outside its sampling", async () => {@@ -866,7 +1010,7 @@ describe("trace batch queue", () => {"authentication","summary",);- return "summarized";+ return { outcome: "summarized", rows: [] };},);const job = {@@ -915,9 +1059,10 @@ describe("trace batch queue", () => {2,{ outcome: "disabled" },);- vi.mocked(summarizeAssembledTrace).mockImplementation(- async () => "unchanged",- );+ vi.mocked(summarizeAssembledTrace).mockImplementation(async () => ({+ outcome: "unchanged",+ rows: [],+ }));});it("assembles each tenant's trace with its latest environment at the boundary and EOF", async () => {diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex 13ae83e8db..da72da5776 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -1,5 +1,6 @@import { type Processor } from "bullmq";import pLimit from "p-limit";+import { backOff } from "exponential-backoff";import { randomUUID } from "node:crypto";import { type Observation } from "@langfuse/shared";import {@@ -18,11 +19,13 @@ import {import {getEnabledTopicsModels,isTopicsProjectEnabled,+ isValidTopicSummary,listTopicFacets,listTopicRules,+ writeTopicSummaries,type TopicsModels,} from "@langfuse/shared/topics/server";-import type { TopicFacet } from "@langfuse/shared/topics";+import type { TopicFacet, TopicSummary } from "@langfuse/shared/topics";import { env } from "../env";import {getDeterministicSamplingValue,@@ -33,14 +36,18 @@ import {pauseForRejectedConnection,TopicsProviderUnavailable,} from "../features/topics/provider-error";+import { TopicMetrics } from "../features/topics/metrics";+import { isRetryableClickhouseInsertError } from "../services/ClickhouseWriter";import { summarizeAssembledTrace } from "../features/topics/summarizeAssembledTrace";import { recordTraceBatchTranscript } from "../features/traceBatching/traceBatchTranscript";type TraceBatchObservation = Observation & { eventTimestamp: string };-type TraceOutcome =+// Rows include a failed trace's valid facets; the batch writes them once.+type TraceOutcome = (| { outcome: "disabled" | "unchanged" | "summarized" }- | { outcome: "failed"; reason: string };+ | { outcome: "failed"; reason: string }+) & { rows: TopicSummary[] };/*** Topics settings of one batch, loaded with one query while the read starts.@@ -126,6 +133,7 @@ async function summarizeTraceBatch(if (row.id !== current.id) return row.id > current.id ? row : current;return row.environment > current.environment ? row : current;});+ let rows: TopicSummary[] = [];try {const project = await topics.get(first.projectId);if (@@ -135,20 +143,21 @@ async function summarizeTraceBatch(samplingRate: project.sampling,}))- return { outcome: "disabled" };- return {- outcome: await summarizeAssembledTrace({- projectId: first.projectId,- traceId,- traceTimestamp,- jobEnqueuedAt,- environment: latest.environment,- traceName: first.name ?? "",- transcript,- models: project.models,- facets: project.facets,- }),- };+ return { outcome: "disabled", rows };+ const result = await summarizeAssembledTrace({+ projectId: first.projectId,+ traceId,+ traceTimestamp,+ jobEnqueuedAt,+ environment: latest.environment,+ traceName: first.name ?? "",+ transcript,+ models: project.models,+ facets: project.facets,+ });+ rows = result.rows;+ if (result.failure === undefined) return { outcome: result.outcome, rows };+ throw result.failure;} catch (error) {// One trace's failure must not fail or re-read the shared batch.const reason =@@ -159,10 +168,47 @@ async function summarizeTraceBatch(traceId,reason,});- return { outcome: "failed", reason };+ return { outcome: "failed", reason, rows };}}+/**+ * Writes the batch's rows in one insert and returns the rows it dropped. Invalid rows are+ * dropped up front so they cannot fail the others; like ClickhouseWriter, transient errors+ * are retried and any other error drops the insert once.+ */+async function writeBatchSummaries(+ rows: TopicSummary[],+): Promise<TopicSummary[]> {+ const valid = rows.filter(isValidTopicSummary);+ const invalid = rows.filter((row) => !isValidTopicSummary(row));+ if (invalid.length > 0)+ logger.warn("Topics summary rows failed validation", {+ rowCount: invalid.length,+ });+ if (valid.length === 0) return invalid;+ try {+ await new TopicMetrics().measure(+ "storage",+ () =>+ backOff(() => writeTopicSummaries(valid), {+ numOfAttempts: env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS,+ retry: isRetryableClickhouseInsertError,+ }),+ "storage",+ );+ } catch (error) {+ logger.warn("Topics summary write failed for trace batch", {+ rowCount: valid.length,+ error: error instanceof Error ? error.message : String(error),+ });+ return rows;+ }+ for (const row of valid)+ recordIncrement("langfuse.topics.facet_summaries", 1, { state: row.state });+ return invalid;+}+/** Failed traces are counted, not retried. */function recordTopicsOutcomes(outcomes: TraceOutcome[]): void {const counts = new Map<string, number>();@@ -249,6 +295,7 @@ export const traceBatchQueueProcessor: Processor<}const batch = event.payload;const topicsOutcomes: TraceOutcome[] = [];+ let droppedRows = new Set<TopicSummary>();const topics = new TopicsBatchProjects(batch.traces.map((trace) => trace.projectId),);@@ -428,8 +475,11 @@ export const traceBatchQueueProcessor: Processor<enqueueSummary(traceObservations, traceBytes);} finally {try {- // Drain accepted summaries even when the stream fails.+ // Drain and store accepted summaries even when the stream fails, so a retry skips them.await Promise.all(pendingSummaries);+ droppedRows = new Set(+ await writeBatchSummaries(topicsOutcomes.flatMap(({ rows }) => rows)),+ );} finally {activeReads--;recordTraceBatchActiveReads();@@ -454,7 +504,14 @@ export const traceBatchQueueProcessor: Processor<batch.traces.length - foundTraces.size,);- recordTopicsOutcomes(topicsOutcomes);+ recordTopicsOutcomes(+ topicsOutcomes.map((result) =>+ result.outcome === "summarized" &&+ result.rows.some((row) => droppedRows.has(row))+ ? { outcome: "failed", reason: "storage", rows: result.rows }+ : result,+ ),+ );// Retries can repeat this read; observation payloads never enter job results.outcome = "success";return {diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex 29c8fb891d..ba4c119d89 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -27,6 +27,20 @@ export { TableName } from "./types";const MULTI_PROJECT_LOG_COMMENT_PROJECT_ID = "MULTI_PROJECT";+export function isRetryableClickhouseInsertError(error: unknown): boolean {+ if (!error || typeof error !== "object") return false;++ const errorMessage = (error as Error).message?.toLowerCase() || "";++ // Socket hang up and client-side request timeouts ("Timeout error." from+ // @clickhouse/client when request_timeout elapses) are transient: a retry+ // opens a fresh connection that can land on a healthy replica.+ return (+ errorMessage.includes("socket hang up") ||+ errorMessage.includes("timeout error")+ );+}+export class ClickhouseWriter<PayloadMap extends WriterPayloadMap = JsonWriterPayloadMap,> {@@ -202,20 +216,6 @@ export class ClickhouseWriter<return flush;}- private isRetryableError(error: unknown): boolean {- if (!error || typeof error !== "object") return false;-- const errorMessage = (error as Error).message?.toLowerCase() || "";-- // Socket hang up and client-side request timeouts ("Timeout error." from- // @clickhouse/client when request_timeout elapses) are transient: a retry- // opens a fresh connection that can land on a healthy replica.- return (- errorMessage.includes("socket hang up") ||- errorMessage.includes("timeout error")- );- }-private isSizeError(error: unknown): boolean {if (!error || typeof error !== "object") return false;@@ -344,7 +344,7 @@ export class ClickhouseWriter<{numOfAttempts: env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS,retry: (error: Error, attemptNumber: number) => {- const isRetryable = this.isRetryableError(error);+ const isRetryable = isRetryableClickhouseInsertError(error);const isSizeError = this.isSizeError(error);const isStringLengthError = this.isStringLengthError(error);