🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): keep trace batch reads streaming while topics run, and trace topics summaries (#18339)
Lysander Kieseld2d5b337e0b4
diff --git a/.env.dev.example b/.env.dev.exampleindex 3b060175ae..2ff8783ff4 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -94,6 +94,10 @@ NEXTAUTH_SECRET="secret"# LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED=true# QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED=true# LANGFUSE_TRACE_BATCH_READ_ENABLED=true+# ClickHouse socket idle timeout during streaming, including summary backpressure.+# LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS=40000+# Queued input/output/metadata bytes; excludes current trace and object overhead.+# LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES=104857600# Unset idle is 2 minutes when NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=DEV, else 10 minutes.LANGFUSE_TRACE_BATCH_IDLE_MS=120000# Summaries and embeddings use Bedrock in LANGFUSE_AI_AWS_BEDROCK_REGION.diff --git a/.env.prod.example b/.env.prod.exampleindex 0c389446e4..703aad6b68 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -530,3 +530,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# Independent per-instance concurrency limits; provisional defaults, tune with load tests.# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128++# Internal cloud trace-batch reader controls; these do not enable processing.+# ClickHouse socket idle timeout during streaming, including summary backpressure.+# LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS=40000+# Queued input/output/metadata bytes; excludes current trace and object overhead.+# LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES=104857600diff --git a/packages/shared/src/server/repositories/trace-batch.test.ts b/packages/shared/src/server/repositories/trace-batch.test.tsindex 365b2f4f46..4127e7820d 100644--- a/packages/shared/src/server/repositories/trace-batch.test.ts+++ b/packages/shared/src/server/repositories/trace-batch.test.ts@@ -92,6 +92,7 @@ describe("trace batch query controls", () => {maxBlockSize,experimentId: "arm-b",queryId: "read-attempt",+ requestTimeoutMs: 40_000,},);expect(await stream.next()).toEqual({ value: row, done: false });@@ -100,6 +101,10 @@ describe("trace batch query controls", () => {expect(sent).toEqual({...baseline,queryId: "read-attempt",+ clickhouseConfigs: {+ ...baseline.clickhouseConfigs,+ request_timeout: 40_000,+ },tags: { ...baseline.tags, experimentId: "arm-b" },clickhouseSettings: {...baseline.clickhouseSettings,diff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.tsindex c7a018c1ed..28c28f9ec2 100644--- a/packages/shared/src/server/repositories/trace-batch.ts+++ b/packages/shared/src/server/repositories/trace-batch.ts@@ -210,6 +210,7 @@ export async function* getTraceBatchEventStream(options: {maxThreads?: number;maxBlockSize?: number;+ requestTimeoutMs?: number;experimentId?: string;queryId?: string;} = {},@@ -229,6 +230,9 @@ export async function* getTraceBatchEventStream(preferredClickhouseService: "EventsReadOnly",clickhouseConfigs: {compression: { response: true },+ ...(options.requestTimeoutMs === undefined+ ? {}+ : { request_timeout: options.requestTimeoutMs }),},// Bound background-read CPU/time; timeouts fail instead of returning partial results.clickhouseSettings: {diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex b9be29f94e..dfcf856de3 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -58,7 +58,10 @@tool-response size uses comparable character counts over message parts.Allowlisted projects are then summarized from that assembled transcript.Allow one pending tokenization promise per batch while- buffering the next trace, and drain it even on read failure. Never flush a failed+ buffering completed traces behind the serial summary chain. Pause reads only+ above the configured queued-payload threshold, then resume as soon as completed+ summaries reduce it to that threshold or below. The client idle timeout must+ account for these pauses. Drain accepted summaries even on read failure. Never flush a failedstream's partial final trace; completion covers the query window, not future arrivals.- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.- Service layer: `src/services/*`diff --git a/worker/src/env.ts b/worker/src/env.tsindex 6aaf1efca0..a23b80adcc 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -134,6 +134,16 @@ const EnvSchema = z.object({.int().positive().default(2),+ LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS: z.coerce+ .number()+ .int()+ .positive()+ .default(40_000),+ LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES: z.coerce+ .number()+ .int()+ .positive()+ .default(100 * 1024 * 1024),LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: z.coerce.number().int()diff --git a/worker/src/features/topics/metrics.ts b/worker/src/features/topics/metrics.tsindex 91b6a4876b..20dddccd41 100644--- a/worker/src/features/topics/metrics.ts+++ b/worker/src/features/topics/metrics.ts@@ -6,6 +6,7 @@ import { TopicsProviderUnavailable } from "./provider-error";type TopicStage =| "execution"+ | "trace"| "transcript"| "summary"| "embedding"diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.tsindex dcdf86d9d5..1663c796f3 100644--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts@@ -1,5 +1,9 @@import { beforeEach, describe, expect, it, vi } from "vitest";-import type { Transcript } from "@langfuse/shared/src/server";+import {+ recordDistribution,+ recordIncrement,+ type Transcript,+} from "@langfuse/shared/src/server";import type { TopicFacet } from "@langfuse/shared/topics";const state = vi.hoisted(() => ({@@ -9,6 +13,7 @@ const state = vi.hoisted(() => ({write: vi.fn(),summarize: vi.fn(),embed: vi.fn(),+ setAttribute: vi.fn(),}));vi.mock("@langfuse/shared/topics/server", () => ({@@ -22,6 +27,17 @@ vi.mock("@langfuse/shared/topics/server", () => ({writeTopicSummaries: (...args: unknown[]) => state.write(...args),TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v2",}));+vi.mock("@langfuse/shared/src/server", () => ({+ recordDistribution: vi.fn(),+ recordIncrement: vi.fn(),+ instrumentAsync: async (+ _options: unknown,+ action: (span: {+ setAttributes: () => void;+ setAttribute: typeof state.setAttribute;+ }) => Promise<unknown>,+ ) => action({ setAttributes: vi.fn(), setAttribute: state.setAttribute }),+}));vi.mock("./models", () => ({summarizeTopicTraceFacets: (...args: unknown[]) => state.summarize(...args),embedTopicSummary: (...args: unknown[]) => state.embed(...args),@@ -81,12 +97,15 @@ const usage = {};beforeEach(() => {+ vi.mocked(recordDistribution).mockClear();+ vi.mocked(recordIncrement).mockClear();state.enabled = true;state.facets.mockReset();state.stored.mockReset();state.write.mockReset();state.summarize.mockReset();state.embed.mockReset();+ state.setAttribute.mockClear();state.facets.mockResolvedValue([facet]);state.stored.mockResolvedValue([]);state.write.mockResolvedValue(undefined);@@ -164,6 +183,11 @@ describe("summarizeAssembledTrace", () => {},});expect(written.costDetails.total).toBeCloseTo(0.00001448, 12);+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.topics.stage_duration_ms",+ expect.any(Number),+ { stage: "trace", outcome: "success", unit: "milliseconds" },+ );state.stored.mockImplementation(async (_projectId, _filter, timeRange) =>new Date(written.unitStartTime) >= timeRange.from &&new Date(written.unitStartTime) < timeRange.to@@ -263,10 +287,122 @@ describe("summarizeAssembledTrace", () => {// The shared call's usage moves to the first row that is written.expect(written[0].usageDetails.summary_input).toBe(20);expect(written[0]).not.toHaveProperty("output");+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.topics.stage_duration_ms",+ expect.any(Number),+ { stage: "summary", outcome: "failed", unit: "milliseconds" },+ );+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(([name]) => name === "langfuse.topics.errors"),+ ).toEqual([+ [+ "langfuse.topics.errors",+ 1,+ { stage: "summary", reason: "invalid_output" },+ ],+ ]);+ expect(state.setAttribute).toHaveBeenCalledWith(+ "langfuse.topics.trace_outcome",+ "failed",+ );+ });++ it("persists successful facets and attributes an embedding failure once", async () => {+ state.facets.mockResolvedValue([+ facet,+ {+ ...facet,+ id: "facet-2",+ name: "Issues",+ versions: [{ ...facet.versions[0], facetId: "facet-2", version: 1 }],+ },+ ]);+ state.summarize.mockResolvedValue({+ output: {+ intent_1: { summary: "Export monthly sales.", status: "applicable" },+ issues_2: { summary: "", status: "not_applicable" },+ },+ ...usage,+ });+ const error = new Error("embedding provider unavailable");+ state.embed.mockRejectedValueOnce(error);+ await expect(+ summarizeAssembledTrace({+ projectId: "project-a",+ traceId: "trace-1",+ traceTimestamp: "2026-09-22T12:00:00.000Z",+ environment: "default",+ traceName: "agent-turn",+ transcript,+ }),+ ).rejects.toBe(error);+ expect(state.summarize).toHaveBeenCalledOnce();+ expect(state.write).toHaveBeenCalledOnce();+ expect(state.write.mock.calls[0][0]).toEqual([+ expect.objectContaining({+ facetId: "facet-2",+ state: "not_applicable",+ usageDetails: usage.usageDetails,+ }),+ ]);+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(([name]) => name === "langfuse.topics.errors"),+ ).toEqual([+ ["langfuse.topics.errors", 1, { stage: "embedding", reason: "unknown" }],+ ]);+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.topics.stage_duration_ms",+ expect.any(Number),+ { stage: "embedding", outcome: "failed", unit: "milliseconds" },+ );+ expect(state.setAttribute).toHaveBeenCalledWith(+ "langfuse.topics.trace_outcome",+ "failed",+ );});it("propagates a provider failure for the trace outcome", async () => {- state.summarize.mockRejectedValue(new Error("provider unavailable"));+ const error = new Error("provider unavailable");+ state.summarize.mockRejectedValue(error);+ await expect(+ summarizeAssembledTrace({+ projectId: "project-a",+ traceId: "trace-1",+ traceTimestamp: "2026-09-22T12:00:00.000Z",+ environment: "default",+ traceName: "agent-turn",+ transcript,+ }),+ ).rejects.toBe(error);+ expect(state.setAttribute).toHaveBeenCalledWith(+ "langfuse.topics.trace_outcome",+ "failed",+ );+ expect(state.write).not.toHaveBeenCalled();+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.topics.stage_duration_ms",+ expect.any(Number),+ { stage: "trace", outcome: "failed", unit: "milliseconds" },+ );+ // Nested stages report one error, attributed to the failing call.+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(([name]) => name === "langfuse.topics.errors"),+ ).toEqual([+ ["langfuse.topics.errors", 1, { stage: "summary", reason: "unknown" }],+ ]);+ });++ it("attributes invalid summary output to the failed summary stage", async () => {+ state.summarize.mockResolvedValue({+ output: { intent_1: { summary: " ", status: "applicable" } },+ ...usage,+ });await expect(summarizeAssembledTrace({projectId: "project-a",@@ -276,7 +412,28 @@ describe("summarizeAssembledTrace", () => {traceName: "agent-turn",transcript,}),- ).rejects.toThrow("provider unavailable");+ ).rejects.toMatchObject({ reason: "invalid_output" });+ expect(state.embed).not.toHaveBeenCalled();expect(state.write).not.toHaveBeenCalled();+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.topics.stage_duration_ms",+ expect.any(Number),+ { stage: "summary", outcome: "failed", unit: "milliseconds" },+ );+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(([name]) => name === "langfuse.topics.errors"),+ ).toEqual([+ [+ "langfuse.topics.errors",+ 1,+ { stage: "summary", reason: "invalid_output" },+ ],+ ]);+ expect(state.setAttribute).toHaveBeenCalledWith(+ "langfuse.topics.trace_outcome",+ "failed",+ );});});diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.tsindex 16103a61e9..947639dffe 100644--- a/worker/src/features/topics/summarizeAssembledTrace.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.ts@@ -11,8 +11,13 @@ import {type TopicFacetVersion,type TopicSummary,} from "@langfuse/shared/topics";-import { recordIncrement, type Transcript } from "@langfuse/shared/src/server";+import {+ instrumentAsync,+ recordIncrement,+ type Transcript,+} from "@langfuse/shared/src/server";import { prepareAssembledTopicTranscript } from "./assembledTranscript";+import { TopicMetrics } from "./metrics";import { embedTopicSummary, summarizeTopicTraceFacets } from "./models";import { TopicsProviderUnavailable } from "./provider-error";import {@@ -33,20 +38,50 @@ const NO_USAGE: TopicModelUsage = {costDetails: {},};-/**- * Summarizes one trace from the transcript the batch job already assembled,- * with one model call for all of its pending facets. Does not load- * observations again.- */-export async function summarizeAssembledTrace(input: {+type AssembledTraceInput = {projectId: string;traceId: string;traceTimestamp: string;environment: string;traceName: string;transcript: Transcript | null;-}): Promise<"disabled" | "unchanged" | "summarized"> {+};++/**+ * Summarizes one trace from the transcript the batch job already assembled,+ * with one model call for all of its pending facets. Does not load+ * observations again.+ */+export async function summarizeAssembledTrace(+ input: AssembledTraceInput,+): Promise<"disabled" | "unchanged" | "summarized"> {if (!isTopicsProjectEnabled(input.projectId)) return "disabled";+ return instrumentAsync(+ { name: "topics-trace-summary", traceScope: "topics" },+ async (span) => {+ span.setAttributes({+ "langfuse.project.id": input.projectId,+ "langfuse.trace.id": input.traceId,+ });+ const metrics = new TopicMetrics();+ try {+ const outcome = await metrics.measure("trace", () =>+ summarizeEnabledTrace(input, metrics),+ );+ span.setAttribute("langfuse.topics.trace_outcome", outcome);+ return outcome;+ } catch (error) {+ span.setAttribute("langfuse.topics.trace_outcome", "failed");+ throw error;+ }+ },+ );+}++async function summarizeEnabledTrace(+ input: AssembledTraceInput,+ metrics: TopicMetrics,+): Promise<"unchanged" | "summarized"> {const models = getTopicsModelConfig();if (!models.summaryModel || !models.embeddingModel)throw new TopicsProviderUnavailable(@@ -94,11 +129,39 @@ export async function summarizeAssembledTrace(input: {key: `${name.toLowerCase().replace(/[^a-z0-9]+/g, "_")}_${index + 1}`,facet: version,}));- const result = await summarizeTopicTraceFacets(- keyed,- prepared.text,- config,- );+ const invalidFacets = new Set<string>();+ let generated:+ | Awaited<ReturnType<typeof summarizeTopicTraceFacets>>+ | undefined;+ const result = await metrics+ .measure("summary", async () => {+ generated = await summarizeTopicTraceFacets(+ keyed,+ prepared.text,+ config,+ );+ const outputs = generated.output as Record<string, FacetOutput>;+ for (const { key } of keyed) {+ const message = outputs[key]+ ? topicSummaryOutputError(outputs[key])+ : "The model response is missing a facet.";+ if (!message) continue;+ const error = new TopicsProviderUnavailable(+ message,+ "invalid_output",+ );+ invalidFacets.add(key);+ metrics.error("summary", error);+ if (failure === undefined) failure = error;+ }+ if (failure !== undefined) throw failure;+ return generated;+ })+ .catch((error: unknown) => {+ // Valid facets can still be persisted after summary validation fails.+ if (error !== failure || !generated) throw error;+ return generated;+ });const outputs = result.output as Record<string, FacetOutput>;const usage: TopicModelUsage = {providedUsageDetails: result.providedUsageDetails,@@ -109,6 +172,7 @@ export async function summarizeAssembledTrace(input: {rows = [];// A failed facet (invalid output, embedding error) must not discard the others.for (const [index, { key }] of keyed.entries()) {+ if (invalidFacets.has(key)) continue;try {rows.push(await finalizeSummary(@@ -118,6 +182,7 @@ export async function summarizeAssembledTrace(input: {rows.length ? NO_USAGE : usage,embeddingConfig.embeddingDimensions,embeddingModel,+ metrics,),);} catch (error) {@@ -125,7 +190,8 @@ export async function summarizeAssembledTrace(input: {}}}- if (rows.length) await writeTopicSummaries(rows);+ if (rows.length)+ await metrics.measure("storage", () => writeTopicSummaries(rows));for (const row of rows)recordIncrement("langfuse.topics.facet_summaries", 1, { state: row.state });if (failure) throw failure;@@ -169,18 +235,12 @@ function baseSummary(async function finalizeSummary(base: TopicSummary,- output: FacetOutput | undefined,+ output: FacetOutput,usage: TopicModelUsage,dimensions: number,embeddingModel: string,+ metrics: TopicMetrics,): Promise<TopicSummary> {- if (!output)- throw new TopicsProviderUnavailable(- "The model response is missing a facet.",- "invalid_output",- );- const error = topicSummaryOutputError(output);- if (error) throw new TopicsProviderUnavailable(error, "invalid_output");if (output.status !== "applicable")return {...base,@@ -191,7 +251,9 @@ async function finalizeSummary(...usage,};const summary = output.summary.trim();- const embedded = await embedTopicSummary(summary, dimensions, embeddingModel);+ const embedded = await metrics.measure("embedding", () =>+ embedTopicSummary(summary, dimensions, embeddingModel),+ );return {...base,state: "complete",diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex df9eb87e44..599de25406 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -24,6 +24,8 @@ templates. The local dev template sets the idle time below.| `LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` | `5000` | Exclude a trace whose pending window exceeds this many event updates; `0` disables. || `LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` | `52428800` | Exclude a trace whose pending window exceeds these serialized bytes (50 MiB); `0` disables. || `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |+| `LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS` | `40000` | ClickHouse client socket idle timeout in milliseconds, including pauses for summary backpressure. Independent of the 30-second server query execution limit. |+| `LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES` | `104857600` | Queued input/output/metadata payload threshold (100 MiB). Includes the active summary; excludes the current trace, tool fields, transport buffers and JavaScript object overhead. || `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. || `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. || `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |@@ -270,19 +272,34 @@ unavailable Topics token estimate increments`topics_transcript_token_estimation_unavailable`; a rejected estimate increments`topics_transcript_failed`. Character metrics remain available in either case.-The worker submits one transcript for tokenization while streaming the next-trace's observations. Before submitting another transcript it awaits the previous-promise. Projections are tokenized sequentially, keeping at most one pending-tokenizer request per batch. Success and stream-failure both drain accepted promises; a stream failure never assembles its-partial final trace. Assembly itself remains synchronous. Wall-clock duration+The reader queues completed traces behind one serial transcript/Topics chain+while continuing to consume ClickHouse rows. Projections are tokenized+sequentially, keeping at most one pending tokenizer request per batch. The chain+yields to the event loop before each trace's assembly. When queued logical+payload bytes exceed `LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES`, reading+pauses until completed summaries reduce the queue to that threshold or below;+it does not wait for the entire chain. Each pause increments+`langfuse.trace_batch.summary_backpressure`. Time from enqueue to summary start+is recorded as `langfuse.trace_batch.summary_queue_wait_ms`.++The dispatcher byte budget estimates only pending updates, while reads can+include older observations, so it does not guarantee avoiding backpressure.+`LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS` allows 40 seconds of socket inactivity+by default. This is a starting value, not a per-trace processing deadline:+slow tokenization or provider calls can still exceed it. Tune the timeout with+observed pauses and provider latency; the query's server execution limit remains+30 seconds. Success and stream failure both drain accepted summaries; a stream+failure never assembles its partial final trace. Assembly itself remains+synchronous. Wall-clock durationof that work, including tokenization, is the span attribute `duration_ms` andthe experiment completion log. The generic queue series`langfuse.queue.trace_batch.time_distribution` with `type:processing` recordsthe same duration when the processor returns.-Memory includes the current trace, the previous transcript being tokenized and-stream buffers, multiplied by active batch jobs. Each trace also retains its+Memory includes the current trace, queued observations, the active transcript+and stream buffers, multiplied by active batch jobs. The payload threshold is+checked at trace boundaries and is not a hard job-memory limit.+The active trace also retains itsassembled JSON, generic text, and Topics text for sequential tokenizer calls.A single trace remains unbounded. Each message belongs to one JSON partition;there is no additional tool-response tokenization pass.diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex e493c07b0e..a52c880cb8 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -31,6 +31,7 @@ import { tokenCount } from "../../features/tokenisation/usage";import { TopicsProviderUnavailable } from "../../features/topics/provider-error";import { summarizeAssembledTrace } from "../../features/topics/summarizeAssembledTrace";import { recordTraceBatchTranscript } from "../../features/traceBatching/traceBatchTranscript";+import * as traceBatchTranscript from "../../features/traceBatching/traceBatchTranscript";import {recordTraceBatchActiveReads,traceBatchQueueProcessor,@@ -440,7 +441,7 @@ describe("trace batch queue", () => {);it.each([false, true])(- "overlaps one tokenization with streaming, bounds pending work and drains it (stream fails: %s)",+ "keeps streaming while one tokenization is pending and drains it (stream fails: %s)",async (streamFails) => {let resolveFirst!: (tokens: number) => void;let rejectSecond!: (error: Error) => void;@@ -478,6 +479,8 @@ describe("trace batch queue", () => {input: "hello",output: "world",metadata: {},+ level: "DEFAULT",+ status_message: null,tool_definitions: {},tool_calls: [],tool_call_names: [],@@ -513,15 +516,16 @@ describe("trace batch queue", () => {);void processing.then(() => (settled = true));try {- // While a is being tokenized, all of b can arrive. At c's boundary,- // processing waits for a before submitting b to the tokenizer pool.- await vi.waitFor(() => expect(suppliedThirdTrace).toBe(true));- expect(reachedEnd).toBe(false);+ // The stream drains while a is being tokenized, so the socket never+ // idles; b and c wait behind a for the shared tokenizer pool.+ await vi.waitFor(() => expect(reachedEnd).toBe(true));+ expect(suppliedThirdTrace).toBe(true);expect(tokenCountAsync).toHaveBeenCalledTimes(1);expect(exporter.getFinishedSpans()).toHaveLength(0);resolveFirst(100);- await vi.waitFor(() => expect(reachedEnd).toBe(true));- expect(tokenCountAsync).toHaveBeenCalledTimes(5);+ await vi.waitFor(() =>+ expect(tokenCountAsync).toHaveBeenCalledTimes(5),+ );expect(settled).toBe(false);// An unavailable token estimate must not retry a whole batch or mask// the original stream failure; its promise is drained before returning.@@ -569,6 +573,98 @@ describe("trace batch queue", () => {}},);+ it("resumes streaming below the payload cap while a later Topics summary is pending", async () => {+ const originalCap = env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES;+ env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = 1_000;+ const payload = "x".repeat(600);+ let reachedEnd = false;+ let resolveFirst!: () => void;+ let resolveSecond!: () => void;+ const first = new Promise<void>((resolve) => (resolveFirst = resolve));+ const second = new Promise<void>((resolve) => (resolveSecond = resolve));+ vi.mocked(summarizeAssembledTrace)+ .mockImplementationOnce(async () => {+ await first;+ return "summarized";+ })+ .mockImplementationOnce(async () => {+ await second;+ return "summarized";+ });+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {+ for (const traceId of ["a", "b", "c"]) {+ yield {+ project_id: "project",+ trace_id: traceId,+ environment: "default",+ span_id: traceId,+ parent_span_id: null,+ start_time: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000000",+ type: "GENERATION",+ name: "generation",+ input: JSON.stringify([{ role: "user", content: "hello" }]),+ output: "",+ metadata: traceId === "c" ? {} : { payload },+ level: "DEFAULT",+ status_message: null,+ tool_definitions: {},+ tool_calls: [],+ tool_call_names: [],+ };+ }+ reachedEnd = true;+ });+ const job = {+ data: {+ id: "summary-backpressure",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(),+ payload: {+ traces: ["a", "b", "c"].map((traceId) => ({+ projectId: "project",+ traceId,+ minStart: 0,+ maxStart: 1,+ revision: "r",+ })),+ },+ },+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ let settled = false;+ const processing = Promise.resolve(+ traceBatchQueueProcessor(job, undefined),+ );+ void processing.then(() => (settled = true));+ try {+ await vi.waitFor(() =>+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.summary_backpressure",+ 1,+ ),+ );+ expect(reachedEnd).toBe(false);+ await vi.waitFor(() =>+ expect(summarizeAssembledTrace).toHaveBeenCalledTimes(1),+ );+ resolveFirst();+ await vi.waitFor(() => expect(reachedEnd).toBe(true));+ await vi.waitFor(() =>+ expect(summarizeAssembledTrace).toHaveBeenCalledTimes(2),+ );+ expect(settled).toBe(false);+ resolveSecond();+ await expect(processing).resolves.toEqual(+ expect.objectContaining({ traceCount: 3 }),+ );+ } finally {+ resolveFirst();+ resolveSecond();+ await processing;+ env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = originalCap;+ vi.mocked(summarizeAssembledTrace).mockResolvedValue("disabled");+ }+ });it("finishes the batch past a failed trace, reports outcomes and does not re-enqueue the failure", async () => {const traces = ["a", "b", "c"];vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {@@ -586,6 +682,8 @@ describe("trace batch queue", () => {input: JSON.stringify([{ role: "user", content: projectId }]),output: JSON.stringify({ role: "assistant", content: "answer" }),metadata: {},+ level: "DEFAULT",+ status_message: null,tool_definitions: {},tool_calls: [],tool_call_names: [],@@ -656,6 +754,8 @@ describe("trace batch queue", () => {input: JSON.stringify([userMessage]),output: JSON.stringify(answer),metadata: {},+ level: "DEFAULT",+ status_message: null,tool_definitions: {},tool_calls: [],tool_call_names: [],@@ -681,8 +781,6 @@ describe("trace batch queue", () => {environment: "staging",type: "SPAN",};- // The first trace is processed before requesting more stream rows.- expect(tokenCountAsync).toHaveBeenCalledTimes(1);yield {...row,project_id: "b",@@ -829,6 +927,7 @@ describe("trace batch queue", () => {Object.assign(env, {LANGFUSE_TRACE_BATCH_MAX_THREADS: 1,LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: 512,+ LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS: 45_000,LANGFUSE_TRACE_BATCH_EXPERIMENT_ID: "arm-b",BUILD_ID: "test-build",});@@ -858,6 +957,8 @@ describe("trace batch queue", () => {input: "input",output: "output",metadata: { é: "界🙂" },+ level: "DEFAULT",+ status_message: null,tool_definitions: {},tool_calls: [],tool_call_names: [],@@ -900,6 +1001,7 @@ describe("trace batch queue", () => {expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload, {maxThreads: 1,maxBlockSize: 512,+ requestTimeoutMs: 45_000,experimentId: "arm-b",queryId: expect.any(String),});@@ -973,11 +1075,88 @@ describe("trace batch queue", () => {originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE =originalEnv.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE;+ env.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS =+ originalEnv.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS;env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID =originalEnv.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;env.BUILD_ID = originalEnv.BUILD_ID;}});+ it.each([false, true])(+ "records partial-read metrics once when transcript processing fails (stream also fails: %s)",+ async (streamFails) => {+ const summaryError = new Error("transcript assembly failed");+ const streamError = new Error("stream failed");+ vi.spyOn(+ traceBatchTranscript,+ "recordTraceBatchTranscript",+ ).mockRejectedValue(summaryError);+ vi.mocked(getTraceBatchEventStream).mockImplementation(+ async function* () {+ for (const traceId of ["a", "b"]) {+ yield {+ project_id: "project",+ trace_id: traceId,+ environment: "default",+ span_id: traceId,+ parent_span_id: null,+ start_time: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000000",+ type: "GENERATION",+ name: "generation",+ input: "hello",+ output: "",+ metadata: {},+ level: "DEFAULT",+ status_message: null,+ tool_definitions: {},+ tool_calls: [],+ tool_call_names: [],+ };+ }+ if (streamFails) throw streamError;+ },+ );+ const job = {+ data: {+ id: "failed-transcript",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(),+ payload: {+ traces: ["a", "b"].map((traceId) => ({+ projectId: "project",+ traceId,+ minStart: 0,+ maxStart: 1,+ revision: "r",+ })),+ },+ },+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(+ streamFails ? streamError : summaryError,+ );+ expect(+ vi+ .mocked(recordDistribution)+ .mock.calls.filter(([name]) =>+ name.startsWith("langfuse.trace_batch.failed_read_"),+ ),+ ).toEqual([+ ["langfuse.trace_batch.failed_read_observation_count", 2],+ ["langfuse.trace_batch.failed_read_input_bytes", 10],+ ]);+ expect(recordDistribution).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.observation_count",+ expect.any(Number),+ );+ expect(summarizeAssembledTrace).not.toHaveBeenCalled();+ expect(recordGauge).toHaveBeenLastCalledWith(+ "langfuse.trace_batch.active_reads",+ 0,+ );+ },+ );it("retains the active read count when overlapping reads succeed or fail", async () => {env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "overlapping";const log = vi.spyOn(logger, "info").mockImplementation(() => logger);@@ -1109,6 +1288,8 @@ describe("trace batch queue", () => {input: "hello",output: "世界",metadata: { é: "界🙂" },+ level: "DEFAULT",+ status_message: null,tool_definitions: {},tool_calls: [],tool_call_names: [],@@ -1232,6 +1413,8 @@ describe("trace batch queue", () => {input: "hello",output: "world",metadata: {},+ level: "DEFAULT",+ status_message: null,tool_definitions: {},tool_calls: [],tool_call_names: [],diff --git a/worker/src/queues/__tests__/traceBatchStream.integration.test.ts b/worker/src/queues/__tests__/traceBatchStream.integration.test.tsnew file mode 100644index 0000000000..43f8b553c8--- /dev/null+++ b/worker/src/queues/__tests__/traceBatchStream.integration.test.ts@@ -0,0 +1,32 @@+import { setTimeout } from "node:timers/promises";+import { describe, expect, it } from "vitest";+import { queryClickhouseStream } from "@langfuse/shared/src/server";++describe("ClickHouse stream backpressure", () => {+ it("allows a paused consumer to finish when its configured idle timeout exceeds the pause", async () => {+ const readWithPause = async (requestTimeoutMs: number) => {+ // A response larger than socket buffers keeps the connection open while+ // the consumer pauses; numbers() avoids creating persistent test data.+ const stream = queryClickhouseStream<{ number: number; payload: string }>(+ {+ query:+ "SELECT number, repeat(toString(number), 1024) AS payload FROM numbers(10000)",+ clickhouseConfigs: { request_timeout: requestTimeoutMs },+ clickhouseSettings: { max_block_size: "256" },+ },+ );+ try {+ expect((await stream.next()).done).toBe(false);+ await setTimeout(1_000);+ let remaining = 0;+ for await (const _row of stream) remaining++;+ return remaining;+ } finally {+ await stream.return(undefined);+ }+ };++ await expect(readWithPause(250)).rejects.toThrow(/aborted|timeout/i);+ await expect(readWithPause(5_000)).resolves.toBe(9_999);+ }, 20_000);+});diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex 9ad1d964e8..4715ef3c16 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -157,6 +157,7 @@ export const traceBatchQueueProcessor: Processor<const queryOptions = {maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,+ requestTimeoutMs: env.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS,experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,queryId: randomUUID(),};@@ -205,20 +206,57 @@ export const traceBatchQueueProcessor: Processor<});}+ // ClickHouse sockets time out when idle, so the reader never waits on+ // transcript or Topics work unless queued payloads exceed the cap.+ // Traces are still summarized one at a time per batch to keep at most+ // one pending request in the tokenizer pool shared with ingestion.+ const pendingSummaries: Promise<void>[] = [];+ let pendingSummaryBytes = 0;+ let summaryFailed = false;+ let summaryError: unknown;+ const enqueueSummary = (observations: Observation[], bytes: number) => {+ pendingSummaryBytes += bytes;+ const enqueuedAt = performance.now();+ const summary = (pendingSummaries.at(-1) ?? Promise.resolve())+ .then(async () => {+ // Let socket reads run between traces' synchronous assembly.+ await new Promise((resolve) => setImmediate(resolve));+ recordDistribution(+ "langfuse.trace_batch.summary_queue_wait_ms",+ performance.now() - enqueuedAt,+ );+ if (!summaryFailed)+ topicsOutcomes.push(await summarizeTraceBatch(observations));+ })+ .catch((error: unknown) => {+ if (!summaryFailed) summaryError = error;+ summaryFailed = true;+ })+ .finally(() => {+ pendingSummaryBytes -= bytes;+ pendingSummaries.shift();+ });+ pendingSummaries.push(summary);+ };activeReads++;- let pendingTokenization: Promise<void> | undefined;try {recordTraceBatchActiveReads();let traceObservations: Observation[] = [];+ let traceBytes = 0;for await (const event of getTraceBatchEventStream(batch, queryOptions)) {observationCount++;foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));foundProjects.add(event.project_id);// Logical UTF-8 payload size, excluding JSON transport and compression.- inputBytes += Buffer.byteLength(event.input);- outputBytes += Buffer.byteLength(event.output);+ const eventInputBytes = Buffer.byteLength(event.input);+ const eventOutputBytes = Buffer.byteLength(event.output);+ let eventBytes = eventInputBytes + eventOutputBytes;+ inputBytes += eventInputBytes;+ outputBytes += eventOutputBytes;for (const [key, value] of Object.entries(event.metadata)) {- metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);+ const bytes = Buffer.byteLength(key) + Buffer.byteLength(value);+ metadataBytes += bytes;+ eventBytes += bytes;}const previous = traceObservations[0];if (@@ -226,16 +264,25 @@ export const traceBatchQueueProcessor: Processor<(previous.projectId !== event.project_id ||previous.traceId !== event.trace_id)) {- // Overlap tokenization with reading the next trace, but allow only- // one pending estimate per batch so queued payloads stay bounded.- await pendingTokenization;- pendingTokenization = summarizeTraceBatch(traceObservations).then(- (result) => {- topicsOutcomes.push(result);- },- );+ if (summaryFailed) break;+ enqueueSummary(traceObservations, traceBytes);traceObservations = [];+ traceBytes = 0;+ if (+ pendingSummaryBytes >+ env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES+ ) {+ recordIncrement("langfuse.trace_batch.summary_backpressure", 1);+ // Resume as soon as completed work brings the queue below the cap.+ while (+ pendingSummaryBytes >+ env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES+ )+ await pendingSummaries[0];+ }+ if (summaryFailed) break;}+ traceBytes += eventBytes;traceObservations.push(convertObservation({...event,@@ -253,32 +300,18 @@ export const traceBatchQueueProcessor: Processor<);}// Reaching EOF completes the last trace; a failed stream must not flush it.- if (traceObservations.length) {- await pendingTokenization;- pendingTokenization = summarizeTraceBatch(traceObservations).then(- (result) => {- topicsOutcomes.push(result);- },- );- }- } catch (error) {- // Only rows consumed before the failure; never count these as successful throughput.- for (const [name, value] of [- ["observation_count", observationCount],- ["input_bytes", inputBytes],- ] as const) {- recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);- }- throw error;+ if (traceObservations.length && !summaryFailed)+ enqueueSummary(traceObservations, traceBytes);} finally {try {- // Drain accepted tokenization promises even when the stream fails.- await pendingTokenization;+ // Drain accepted summaries even when the stream fails.+ await pendingSummaries.at(-1);} finally {activeReads--;recordTraceBatchActiveReads();}}+ if (summaryFailed) throw summaryError;recordDistribution("langfuse.trace_batch.observation_count",@@ -310,6 +343,15 @@ export const traceBatchQueueProcessor: Processor<ioMetadataBytes,};} finally {+ if (queryId && outcome === "failure") {+ // Consumed rows from failed attempts are not successful throughput.+ for (const [name, value] of [+ ["observation_count", observationCount],+ ["input_bytes", inputBytes],+ ] as const) {+ recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);+ }+ }const durationMs = performance.now() - startedAt;span?.setAttributes({"langfuse.trace_batch.outcome": outcome,