Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): keep trace batch reads streaming while topics run, and trace topics summaries (#18339)

Lysander Kieseld2d5b337e0b4
diff --git a/.env.dev.example b/.env.dev.example
index 3b060175ae..2ff8783ff4 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -94,6 +94,10 @@ NEXTAUTH_SECRET="secret"
# LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED=true
# QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED=true
# LANGFUSE_TRACE_BATCH_READ_ENABLED=true
+# ClickHouse socket idle timeout during streaming, including summary backpressure.
+# LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS=40000
+# Queued input/output/metadata bytes; excludes current trace and object overhead.
+# LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES=104857600
# Unset idle is 2 minutes when NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=DEV, else 10 minutes.
LANGFUSE_TRACE_BATCH_IDLE_MS=120000
# Summaries and embeddings use Bedrock in LANGFUSE_AI_AWS_BEDROCK_REGION.
diff --git a/.env.prod.example b/.env.prod.example
index 0c389446e4..703aad6b68 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -530,3 +530,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# Independent per-instance concurrency limits; provisional defaults, tune with load tests.
# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128
# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128
+
+# Internal cloud trace-batch reader controls; these do not enable processing.
+# ClickHouse socket idle timeout during streaming, including summary backpressure.
+# LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS=40000
+# Queued input/output/metadata bytes; excludes current trace and object overhead.
+# LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES=104857600
diff --git a/packages/shared/src/server/repositories/trace-batch.test.ts b/packages/shared/src/server/repositories/trace-batch.test.ts
index 365b2f4f46..4127e7820d 100644
--- a/packages/shared/src/server/repositories/trace-batch.test.ts
+++ b/packages/shared/src/server/repositories/trace-batch.test.ts
@@ -92,6 +92,7 @@ describe("trace batch query controls", () => {
maxBlockSize,
experimentId: "arm-b",
queryId: "read-attempt",
+ requestTimeoutMs: 40_000,
},
);
expect(await stream.next()).toEqual({ value: row, done: false });
@@ -100,6 +101,10 @@ describe("trace batch query controls", () => {
expect(sent).toEqual({
...baseline,
queryId: "read-attempt",
+ clickhouseConfigs: {
+ ...baseline.clickhouseConfigs,
+ request_timeout: 40_000,
+ },
tags: { ...baseline.tags, experimentId: "arm-b" },
clickhouseSettings: {
...baseline.clickhouseSettings,
diff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.ts
index c7a018c1ed..28c28f9ec2 100644
--- a/packages/shared/src/server/repositories/trace-batch.ts
+++ b/packages/shared/src/server/repositories/trace-batch.ts
@@ -210,6 +210,7 @@ export async function* getTraceBatchEventStream(
options: {
maxThreads?: number;
maxBlockSize?: number;
+ requestTimeoutMs?: number;
experimentId?: string;
queryId?: string;
} = {},
@@ -229,6 +230,9 @@ export async function* getTraceBatchEventStream(
preferredClickhouseService: "EventsReadOnly",
clickhouseConfigs: {
compression: { response: true },
+ ...(options.requestTimeoutMs === undefined
+ ? {}
+ : { request_timeout: options.requestTimeoutMs }),
},
// Bound background-read CPU/time; timeouts fail instead of returning partial results.
clickhouseSettings: {
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index b9be29f94e..dfcf856de3 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -58,7 +58,10 @@
tool-response size uses comparable character counts over message parts.
Allowlisted projects are then summarized from that assembled transcript.
Allow one pending tokenization promise per batch while
- buffering the next trace, and drain it even on read failure. Never flush a failed
+ buffering completed traces behind the serial summary chain. Pause reads only
+ above the configured queued-payload threshold, then resume as soon as completed
+ summaries reduce it to that threshold or below. The client idle timeout must
+ account for these pauses. Drain accepted summaries even on read failure. Never flush a failed
stream's partial final trace; completion covers the query window, not future arrivals.
- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.
- Service layer: `src/services/*`
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 6aaf1efca0..a23b80adcc 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -134,6 +134,16 @@ const EnvSchema = z.object({
.int()
.positive()
.default(2),
+ LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(40_000),
+ LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(100 * 1024 * 1024),
LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: z.coerce
.number()
.int()
diff --git a/worker/src/features/topics/metrics.ts b/worker/src/features/topics/metrics.ts
index 91b6a4876b..20dddccd41 100644
--- a/worker/src/features/topics/metrics.ts
+++ b/worker/src/features/topics/metrics.ts
@@ -6,6 +6,7 @@ import { TopicsProviderUnavailable } from "./provider-error";
type TopicStage =
| "execution"
+ | "trace"
| "transcript"
| "summary"
| "embedding"
diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.ts
index dcdf86d9d5..1663c796f3 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts
@@ -1,5 +1,9 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
-import type { Transcript } from "@langfuse/shared/src/server";
+import {
+ recordDistribution,
+ recordIncrement,
+ type Transcript,
+} from "@langfuse/shared/src/server";
import type { TopicFacet } from "@langfuse/shared/topics";
const state = vi.hoisted(() => ({
@@ -9,6 +13,7 @@ const state = vi.hoisted(() => ({
write: vi.fn(),
summarize: vi.fn(),
embed: vi.fn(),
+ setAttribute: vi.fn(),
}));
vi.mock("@langfuse/shared/topics/server", () => ({
@@ -22,6 +27,17 @@ vi.mock("@langfuse/shared/topics/server", () => ({
writeTopicSummaries: (...args: unknown[]) => state.write(...args),
TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v2",
}));
+vi.mock("@langfuse/shared/src/server", () => ({
+ recordDistribution: vi.fn(),
+ recordIncrement: vi.fn(),
+ instrumentAsync: async (
+ _options: unknown,
+ action: (span: {
+ setAttributes: () => void;
+ setAttribute: typeof state.setAttribute;
+ }) => Promise<unknown>,
+ ) => action({ setAttributes: vi.fn(), setAttribute: state.setAttribute }),
+}));
vi.mock("./models", () => ({
summarizeTopicTraceFacets: (...args: unknown[]) => state.summarize(...args),
embedTopicSummary: (...args: unknown[]) => state.embed(...args),
@@ -81,12 +97,15 @@ const usage = {
};
beforeEach(() => {
+ vi.mocked(recordDistribution).mockClear();
+ vi.mocked(recordIncrement).mockClear();
state.enabled = true;
state.facets.mockReset();
state.stored.mockReset();
state.write.mockReset();
state.summarize.mockReset();
state.embed.mockReset();
+ state.setAttribute.mockClear();
state.facets.mockResolvedValue([facet]);
state.stored.mockResolvedValue([]);
state.write.mockResolvedValue(undefined);
@@ -164,6 +183,11 @@ describe("summarizeAssembledTrace", () => {
},
});
expect(written.costDetails.total).toBeCloseTo(0.00001448, 12);
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.topics.stage_duration_ms",
+ expect.any(Number),
+ { stage: "trace", outcome: "success", unit: "milliseconds" },
+ );
state.stored.mockImplementation(async (_projectId, _filter, timeRange) =>
new Date(written.unitStartTime) >= timeRange.from &&
new Date(written.unitStartTime) < timeRange.to
@@ -263,10 +287,122 @@ describe("summarizeAssembledTrace", () => {
// The shared call's usage moves to the first row that is written.
expect(written[0].usageDetails.summary_input).toBe(20);
expect(written[0]).not.toHaveProperty("output");
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.topics.stage_duration_ms",
+ expect.any(Number),
+ { stage: "summary", outcome: "failed", unit: "milliseconds" },
+ );
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(([name]) => name === "langfuse.topics.errors"),
+ ).toEqual([
+ [
+ "langfuse.topics.errors",
+ 1,
+ { stage: "summary", reason: "invalid_output" },
+ ],
+ ]);
+ expect(state.setAttribute).toHaveBeenCalledWith(
+ "langfuse.topics.trace_outcome",
+ "failed",
+ );
+ });
+
+ it("persists successful facets and attributes an embedding failure once", async () => {
+ state.facets.mockResolvedValue([
+ facet,
+ {
+ ...facet,
+ id: "facet-2",
+ name: "Issues",
+ versions: [{ ...facet.versions[0], facetId: "facet-2", version: 1 }],
+ },
+ ]);
+ state.summarize.mockResolvedValue({
+ output: {
+ intent_1: { summary: "Export monthly sales.", status: "applicable" },
+ issues_2: { summary: "", status: "not_applicable" },
+ },
+ ...usage,
+ });
+ const error = new Error("embedding provider unavailable");
+ state.embed.mockRejectedValueOnce(error);
+ await expect(
+ summarizeAssembledTrace({
+ projectId: "project-a",
+ traceId: "trace-1",
+ traceTimestamp: "2026-09-22T12:00:00.000Z",
+ environment: "default",
+ traceName: "agent-turn",
+ transcript,
+ }),
+ ).rejects.toBe(error);
+ expect(state.summarize).toHaveBeenCalledOnce();
+ expect(state.write).toHaveBeenCalledOnce();
+ expect(state.write.mock.calls[0][0]).toEqual([
+ expect.objectContaining({
+ facetId: "facet-2",
+ state: "not_applicable",
+ usageDetails: usage.usageDetails,
+ }),
+ ]);
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(([name]) => name === "langfuse.topics.errors"),
+ ).toEqual([
+ ["langfuse.topics.errors", 1, { stage: "embedding", reason: "unknown" }],
+ ]);
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.topics.stage_duration_ms",
+ expect.any(Number),
+ { stage: "embedding", outcome: "failed", unit: "milliseconds" },
+ );
+ expect(state.setAttribute).toHaveBeenCalledWith(
+ "langfuse.topics.trace_outcome",
+ "failed",
+ );
});
it("propagates a provider failure for the trace outcome", async () => {
- state.summarize.mockRejectedValue(new Error("provider unavailable"));
+ const error = new Error("provider unavailable");
+ state.summarize.mockRejectedValue(error);
+ await expect(
+ summarizeAssembledTrace({
+ projectId: "project-a",
+ traceId: "trace-1",
+ traceTimestamp: "2026-09-22T12:00:00.000Z",
+ environment: "default",
+ traceName: "agent-turn",
+ transcript,
+ }),
+ ).rejects.toBe(error);
+ expect(state.setAttribute).toHaveBeenCalledWith(
+ "langfuse.topics.trace_outcome",
+ "failed",
+ );
+ expect(state.write).not.toHaveBeenCalled();
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.topics.stage_duration_ms",
+ expect.any(Number),
+ { stage: "trace", outcome: "failed", unit: "milliseconds" },
+ );
+ // Nested stages report one error, attributed to the failing call.
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(([name]) => name === "langfuse.topics.errors"),
+ ).toEqual([
+ ["langfuse.topics.errors", 1, { stage: "summary", reason: "unknown" }],
+ ]);
+ });
+
+ it("attributes invalid summary output to the failed summary stage", async () => {
+ state.summarize.mockResolvedValue({
+ output: { intent_1: { summary: " ", status: "applicable" } },
+ ...usage,
+ });
await expect(
summarizeAssembledTrace({
projectId: "project-a",
@@ -276,7 +412,28 @@ describe("summarizeAssembledTrace", () => {
traceName: "agent-turn",
transcript,
}),
- ).rejects.toThrow("provider unavailable");
+ ).rejects.toMatchObject({ reason: "invalid_output" });
+ expect(state.embed).not.toHaveBeenCalled();
expect(state.write).not.toHaveBeenCalled();
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.topics.stage_duration_ms",
+ expect.any(Number),
+ { stage: "summary", outcome: "failed", unit: "milliseconds" },
+ );
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(([name]) => name === "langfuse.topics.errors"),
+ ).toEqual([
+ [
+ "langfuse.topics.errors",
+ 1,
+ { stage: "summary", reason: "invalid_output" },
+ ],
+ ]);
+ expect(state.setAttribute).toHaveBeenCalledWith(
+ "langfuse.topics.trace_outcome",
+ "failed",
+ );
});
});
diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.ts
index 16103a61e9..947639dffe 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.ts
@@ -11,8 +11,13 @@ import {
type TopicFacetVersion,
type TopicSummary,
} from "@langfuse/shared/topics";
-import { recordIncrement, type Transcript } from "@langfuse/shared/src/server";
+import {
+ instrumentAsync,
+ recordIncrement,
+ type Transcript,
+} from "@langfuse/shared/src/server";
import { prepareAssembledTopicTranscript } from "./assembledTranscript";
+import { TopicMetrics } from "./metrics";
import { embedTopicSummary, summarizeTopicTraceFacets } from "./models";
import { TopicsProviderUnavailable } from "./provider-error";
import {
@@ -33,20 +38,50 @@ const NO_USAGE: TopicModelUsage = {
costDetails: {},
};
-/**
- * Summarizes one trace from the transcript the batch job already assembled,
- * with one model call for all of its pending facets. Does not load
- * observations again.
- */
-export async function summarizeAssembledTrace(input: {
+type AssembledTraceInput = {
projectId: string;
traceId: string;
traceTimestamp: string;
environment: string;
traceName: string;
transcript: Transcript | null;
-}): Promise<"disabled" | "unchanged" | "summarized"> {
+};
+
+/**
+ * Summarizes one trace from the transcript the batch job already assembled,
+ * with one model call for all of its pending facets. Does not load
+ * observations again.
+ */
+export async function summarizeAssembledTrace(
+ input: AssembledTraceInput,
+): Promise<"disabled" | "unchanged" | "summarized"> {
if (!isTopicsProjectEnabled(input.projectId)) return "disabled";
+ return instrumentAsync(
+ { name: "topics-trace-summary", traceScope: "topics" },
+ async (span) => {
+ span.setAttributes({
+ "langfuse.project.id": input.projectId,
+ "langfuse.trace.id": input.traceId,
+ });
+ const metrics = new TopicMetrics();
+ try {
+ const outcome = await metrics.measure("trace", () =>
+ summarizeEnabledTrace(input, metrics),
+ );
+ span.setAttribute("langfuse.topics.trace_outcome", outcome);
+ return outcome;
+ } catch (error) {
+ span.setAttribute("langfuse.topics.trace_outcome", "failed");
+ throw error;
+ }
+ },
+ );
+}
+
+async function summarizeEnabledTrace(
+ input: AssembledTraceInput,
+ metrics: TopicMetrics,
+): Promise<"unchanged" | "summarized"> {
const models = getTopicsModelConfig();
if (!models.summaryModel || !models.embeddingModel)
throw new TopicsProviderUnavailable(
@@ -94,11 +129,39 @@ export async function summarizeAssembledTrace(input: {
key: `${name.toLowerCase().replace(/[^a-z0-9]+/g, "_")}_${index + 1}`,
facet: version,
}));
- const result = await summarizeTopicTraceFacets(
- keyed,
- prepared.text,
- config,
- );
+ const invalidFacets = new Set<string>();
+ let generated:
+ | Awaited<ReturnType<typeof summarizeTopicTraceFacets>>
+ | undefined;
+ const result = await metrics
+ .measure("summary", async () => {
+ generated = await summarizeTopicTraceFacets(
+ keyed,
+ prepared.text,
+ config,
+ );
+ const outputs = generated.output as Record<string, FacetOutput>;
+ for (const { key } of keyed) {
+ const message = outputs[key]
+ ? topicSummaryOutputError(outputs[key])
+ : "The model response is missing a facet.";
+ if (!message) continue;
+ const error = new TopicsProviderUnavailable(
+ message,
+ "invalid_output",
+ );
+ invalidFacets.add(key);
+ metrics.error("summary", error);
+ if (failure === undefined) failure = error;
+ }
+ if (failure !== undefined) throw failure;
+ return generated;
+ })
+ .catch((error: unknown) => {
+ // Valid facets can still be persisted after summary validation fails.
+ if (error !== failure || !generated) throw error;
+ return generated;
+ });
const outputs = result.output as Record<string, FacetOutput>;
const usage: TopicModelUsage = {
providedUsageDetails: result.providedUsageDetails,
@@ -109,6 +172,7 @@ export async function summarizeAssembledTrace(input: {
rows = [];
// A failed facet (invalid output, embedding error) must not discard the others.
for (const [index, { key }] of keyed.entries()) {
+ if (invalidFacets.has(key)) continue;
try {
rows.push(
await finalizeSummary(
@@ -118,6 +182,7 @@ export async function summarizeAssembledTrace(input: {
rows.length ? NO_USAGE : usage,
embeddingConfig.embeddingDimensions,
embeddingModel,
+ metrics,
),
);
} catch (error) {
@@ -125,7 +190,8 @@ export async function summarizeAssembledTrace(input: {
}
}
}
- if (rows.length) await writeTopicSummaries(rows);
+ if (rows.length)
+ await metrics.measure("storage", () => writeTopicSummaries(rows));
for (const row of rows)
recordIncrement("langfuse.topics.facet_summaries", 1, { state: row.state });
if (failure) throw failure;
@@ -169,18 +235,12 @@ function baseSummary(
async function finalizeSummary(
base: TopicSummary,
- output: FacetOutput | undefined,
+ output: FacetOutput,
usage: TopicModelUsage,
dimensions: number,
embeddingModel: string,
+ metrics: TopicMetrics,
): Promise<TopicSummary> {
- if (!output)
- throw new TopicsProviderUnavailable(
- "The model response is missing a facet.",
- "invalid_output",
- );
- const error = topicSummaryOutputError(output);
- if (error) throw new TopicsProviderUnavailable(error, "invalid_output");
if (output.status !== "applicable")
return {
...base,
@@ -191,7 +251,9 @@ async function finalizeSummary(
...usage,
};
const summary = output.summary.trim();
- const embedded = await embedTopicSummary(summary, dimensions, embeddingModel);
+ const embedded = await metrics.measure("embedding", () =>
+ embedTopicSummary(summary, dimensions, embeddingModel),
+ );
return {
...base,
state: "complete",
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index df9eb87e44..599de25406 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -24,6 +24,8 @@ templates. The local dev template sets the idle time below.
| `LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` | `5000` | Exclude a trace whose pending window exceeds this many event updates; `0` disables. |
| `LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` | `52428800` | Exclude a trace whose pending window exceeds these serialized bytes (50 MiB); `0` disables. |
| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |
+| `LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS` | `40000` | ClickHouse client socket idle timeout in milliseconds, including pauses for summary backpressure. Independent of the 30-second server query execution limit. |
+| `LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES` | `104857600` | Queued input/output/metadata payload threshold (100 MiB). Includes the active summary; excludes the current trace, tool fields, transport buffers and JavaScript object overhead. |
| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
@@ -270,19 +272,34 @@ unavailable Topics token estimate increments
`topics_transcript_token_estimation_unavailable`; a rejected estimate increments
`topics_transcript_failed`. Character metrics remain available in either case.
-The worker submits one transcript for tokenization while streaming the next
-trace's observations. Before submitting another transcript it awaits the previous
-promise. Projections are tokenized sequentially, keeping at most one pending
-tokenizer request per batch. Success and stream
-failure both drain accepted promises; a stream failure never assembles its
-partial final trace. Assembly itself remains synchronous. Wall-clock duration
+The reader queues completed traces behind one serial transcript/Topics chain
+while continuing to consume ClickHouse rows. Projections are tokenized
+sequentially, keeping at most one pending tokenizer request per batch. The chain
+yields to the event loop before each trace's assembly. When queued logical
+payload bytes exceed `LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES`, reading
+pauses until completed summaries reduce the queue to that threshold or below;
+it does not wait for the entire chain. Each pause increments
+`langfuse.trace_batch.summary_backpressure`. Time from enqueue to summary start
+is recorded as `langfuse.trace_batch.summary_queue_wait_ms`.
+
+The dispatcher byte budget estimates only pending updates, while reads can
+include older observations, so it does not guarantee avoiding backpressure.
+`LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS` allows 40 seconds of socket inactivity
+by default. This is a starting value, not a per-trace processing deadline:
+slow tokenization or provider calls can still exceed it. Tune the timeout with
+observed pauses and provider latency; the query's server execution limit remains
+30 seconds. Success and stream failure both drain accepted summaries; a stream
+failure never assembles its partial final trace. Assembly itself remains
+synchronous. Wall-clock duration
of that work, including tokenization, is the span attribute `duration_ms` and
the experiment completion log. The generic queue series
`langfuse.queue.trace_batch.time_distribution` with `type:processing` records
the same duration when the processor returns.
-Memory includes the current trace, the previous transcript being tokenized and
-stream buffers, multiplied by active batch jobs. Each trace also retains its
+Memory includes the current trace, queued observations, the active transcript
+and stream buffers, multiplied by active batch jobs. The payload threshold is
+checked at trace boundaries and is not a hard job-memory limit.
+The active trace also retains its
assembled JSON, generic text, and Topics text for sequential tokenizer calls.
A single trace remains unbounded. Each message belongs to one JSON partition;
there is no additional tool-response tokenization pass.
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index e493c07b0e..a52c880cb8 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -31,6 +31,7 @@ import { tokenCount } from "../../features/tokenisation/usage";
import { TopicsProviderUnavailable } from "../../features/topics/provider-error";
import { summarizeAssembledTrace } from "../../features/topics/summarizeAssembledTrace";
import { recordTraceBatchTranscript } from "../../features/traceBatching/traceBatchTranscript";
+import * as traceBatchTranscript from "../../features/traceBatching/traceBatchTranscript";
import {
recordTraceBatchActiveReads,
traceBatchQueueProcessor,
@@ -440,7 +441,7 @@ describe("trace batch queue", () => {
);
it.each([false, true])(
- "overlaps one tokenization with streaming, bounds pending work and drains it (stream fails: %s)",
+ "keeps streaming while one tokenization is pending and drains it (stream fails: %s)",
async (streamFails) => {
let resolveFirst!: (tokens: number) => void;
let rejectSecond!: (error: Error) => void;
@@ -478,6 +479,8 @@ describe("trace batch queue", () => {
input: "hello",
output: "world",
metadata: {},
+ level: "DEFAULT",
+ status_message: null,
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
@@ -513,15 +516,16 @@ describe("trace batch queue", () => {
);
void processing.then(() => (settled = true));
try {
- // While a is being tokenized, all of b can arrive. At c's boundary,
- // processing waits for a before submitting b to the tokenizer pool.
- await vi.waitFor(() => expect(suppliedThirdTrace).toBe(true));
- expect(reachedEnd).toBe(false);
+ // The stream drains while a is being tokenized, so the socket never
+ // idles; b and c wait behind a for the shared tokenizer pool.
+ await vi.waitFor(() => expect(reachedEnd).toBe(true));
+ expect(suppliedThirdTrace).toBe(true);
expect(tokenCountAsync).toHaveBeenCalledTimes(1);
expect(exporter.getFinishedSpans()).toHaveLength(0);
resolveFirst(100);
- await vi.waitFor(() => expect(reachedEnd).toBe(true));
- expect(tokenCountAsync).toHaveBeenCalledTimes(5);
+ await vi.waitFor(() =>
+ expect(tokenCountAsync).toHaveBeenCalledTimes(5),
+ );
expect(settled).toBe(false);
// An unavailable token estimate must not retry a whole batch or mask
// the original stream failure; its promise is drained before returning.
@@ -569,6 +573,98 @@ describe("trace batch queue", () => {
}
},
);
+ it("resumes streaming below the payload cap while a later Topics summary is pending", async () => {
+ const originalCap = env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES;
+ env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = 1_000;
+ const payload = "x".repeat(600);
+ let reachedEnd = false;
+ let resolveFirst!: () => void;
+ let resolveSecond!: () => void;
+ const first = new Promise<void>((resolve) => (resolveFirst = resolve));
+ const second = new Promise<void>((resolve) => (resolveSecond = resolve));
+ vi.mocked(summarizeAssembledTrace)
+ .mockImplementationOnce(async () => {
+ await first;
+ return "summarized";
+ })
+ .mockImplementationOnce(async () => {
+ await second;
+ return "summarized";
+ });
+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
+ for (const traceId of ["a", "b", "c"]) {
+ yield {
+ project_id: "project",
+ trace_id: traceId,
+ environment: "default",
+ span_id: traceId,
+ parent_span_id: null,
+ start_time: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000000",
+ type: "GENERATION",
+ name: "generation",
+ input: JSON.stringify([{ role: "user", content: "hello" }]),
+ output: "",
+ metadata: traceId === "c" ? {} : { payload },
+ level: "DEFAULT",
+ status_message: null,
+ tool_definitions: {},
+ tool_calls: [],
+ tool_call_names: [],
+ };
+ }
+ reachedEnd = true;
+ });
+ const job = {
+ data: {
+ id: "summary-backpressure",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(),
+ payload: {
+ traces: ["a", "b", "c"].map((traceId) => ({
+ projectId: "project",
+ traceId,
+ minStart: 0,
+ maxStart: 1,
+ revision: "r",
+ })),
+ },
+ },
+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+ let settled = false;
+ const processing = Promise.resolve(
+ traceBatchQueueProcessor(job, undefined),
+ );
+ void processing.then(() => (settled = true));
+ try {
+ await vi.waitFor(() =>
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.summary_backpressure",
+ 1,
+ ),
+ );
+ expect(reachedEnd).toBe(false);
+ await vi.waitFor(() =>
+ expect(summarizeAssembledTrace).toHaveBeenCalledTimes(1),
+ );
+ resolveFirst();
+ await vi.waitFor(() => expect(reachedEnd).toBe(true));
+ await vi.waitFor(() =>
+ expect(summarizeAssembledTrace).toHaveBeenCalledTimes(2),
+ );
+ expect(settled).toBe(false);
+ resolveSecond();
+ await expect(processing).resolves.toEqual(
+ expect.objectContaining({ traceCount: 3 }),
+ );
+ } finally {
+ resolveFirst();
+ resolveSecond();
+ await processing;
+ env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = originalCap;
+ vi.mocked(summarizeAssembledTrace).mockResolvedValue("disabled");
+ }
+ });
it("finishes the batch past a failed trace, reports outcomes and does not re-enqueue the failure", async () => {
const traces = ["a", "b", "c"];
vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
@@ -586,6 +682,8 @@ describe("trace batch queue", () => {
input: JSON.stringify([{ role: "user", content: projectId }]),
output: JSON.stringify({ role: "assistant", content: "answer" }),
metadata: {},
+ level: "DEFAULT",
+ status_message: null,
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
@@ -656,6 +754,8 @@ describe("trace batch queue", () => {
input: JSON.stringify([userMessage]),
output: JSON.stringify(answer),
metadata: {},
+ level: "DEFAULT",
+ status_message: null,
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
@@ -681,8 +781,6 @@ describe("trace batch queue", () => {
environment: "staging",
type: "SPAN",
};
- // The first trace is processed before requesting more stream rows.
- expect(tokenCountAsync).toHaveBeenCalledTimes(1);
yield {
...row,
project_id: "b",
@@ -829,6 +927,7 @@ describe("trace batch queue", () => {
Object.assign(env, {
LANGFUSE_TRACE_BATCH_MAX_THREADS: 1,
LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: 512,
+ LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS: 45_000,
LANGFUSE_TRACE_BATCH_EXPERIMENT_ID: "arm-b",
BUILD_ID: "test-build",
});
@@ -858,6 +957,8 @@ describe("trace batch queue", () => {
input: "input",
output: "output",
metadata: { é: "界🙂" },
+ level: "DEFAULT",
+ status_message: null,
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
@@ -900,6 +1001,7 @@ describe("trace batch queue", () => {
expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload, {
maxThreads: 1,
maxBlockSize: 512,
+ requestTimeoutMs: 45_000,
experimentId: "arm-b",
queryId: expect.any(String),
});
@@ -973,11 +1075,88 @@ describe("trace batch queue", () => {
originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;
env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE =
originalEnv.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE;
+ env.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS =
+ originalEnv.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS;
env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID =
originalEnv.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;
env.BUILD_ID = originalEnv.BUILD_ID;
}
});
+ it.each([false, true])(
+ "records partial-read metrics once when transcript processing fails (stream also fails: %s)",
+ async (streamFails) => {
+ const summaryError = new Error("transcript assembly failed");
+ const streamError = new Error("stream failed");
+ vi.spyOn(
+ traceBatchTranscript,
+ "recordTraceBatchTranscript",
+ ).mockRejectedValue(summaryError);
+ vi.mocked(getTraceBatchEventStream).mockImplementation(
+ async function* () {
+ for (const traceId of ["a", "b"]) {
+ yield {
+ project_id: "project",
+ trace_id: traceId,
+ environment: "default",
+ span_id: traceId,
+ parent_span_id: null,
+ start_time: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000000",
+ type: "GENERATION",
+ name: "generation",
+ input: "hello",
+ output: "",
+ metadata: {},
+ level: "DEFAULT",
+ status_message: null,
+ tool_definitions: {},
+ tool_calls: [],
+ tool_call_names: [],
+ };
+ }
+ if (streamFails) throw streamError;
+ },
+ );
+ const job = {
+ data: {
+ id: "failed-transcript",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(),
+ payload: {
+ traces: ["a", "b"].map((traceId) => ({
+ projectId: "project",
+ traceId,
+ minStart: 0,
+ maxStart: 1,
+ revision: "r",
+ })),
+ },
+ },
+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+ await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(
+ streamFails ? streamError : summaryError,
+ );
+ expect(
+ vi
+ .mocked(recordDistribution)
+ .mock.calls.filter(([name]) =>
+ name.startsWith("langfuse.trace_batch.failed_read_"),
+ ),
+ ).toEqual([
+ ["langfuse.trace_batch.failed_read_observation_count", 2],
+ ["langfuse.trace_batch.failed_read_input_bytes", 10],
+ ]);
+ expect(recordDistribution).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.observation_count",
+ expect.any(Number),
+ );
+ expect(summarizeAssembledTrace).not.toHaveBeenCalled();
+ expect(recordGauge).toHaveBeenLastCalledWith(
+ "langfuse.trace_batch.active_reads",
+ 0,
+ );
+ },
+ );
it("retains the active read count when overlapping reads succeed or fail", async () => {
env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "overlapping";
const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
@@ -1109,6 +1288,8 @@ describe("trace batch queue", () => {
input: "hello",
output: "世界",
metadata: { é: "界🙂" },
+ level: "DEFAULT",
+ status_message: null,
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
@@ -1232,6 +1413,8 @@ describe("trace batch queue", () => {
input: "hello",
output: "world",
metadata: {},
+ level: "DEFAULT",
+ status_message: null,
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
diff --git a/worker/src/queues/__tests__/traceBatchStream.integration.test.ts b/worker/src/queues/__tests__/traceBatchStream.integration.test.ts
new file mode 100644
index 0000000000..43f8b553c8
--- /dev/null
+++ b/worker/src/queues/__tests__/traceBatchStream.integration.test.ts
@@ -0,0 +1,32 @@
+import { setTimeout } from "node:timers/promises";
+import { describe, expect, it } from "vitest";
+import { queryClickhouseStream } from "@langfuse/shared/src/server";
+
+describe("ClickHouse stream backpressure", () => {
+ it("allows a paused consumer to finish when its configured idle timeout exceeds the pause", async () => {
+ const readWithPause = async (requestTimeoutMs: number) => {
+ // A response larger than socket buffers keeps the connection open while
+ // the consumer pauses; numbers() avoids creating persistent test data.
+ const stream = queryClickhouseStream<{ number: number; payload: string }>(
+ {
+ query:
+ "SELECT number, repeat(toString(number), 1024) AS payload FROM numbers(10000)",
+ clickhouseConfigs: { request_timeout: requestTimeoutMs },
+ clickhouseSettings: { max_block_size: "256" },
+ },
+ );
+ try {
+ expect((await stream.next()).done).toBe(false);
+ await setTimeout(1_000);
+ let remaining = 0;
+ for await (const _row of stream) remaining++;
+ return remaining;
+ } finally {
+ await stream.return(undefined);
+ }
+ };
+
+ await expect(readWithPause(250)).rejects.toThrow(/aborted|timeout/i);
+ await expect(readWithPause(5_000)).resolves.toBe(9_999);
+ }, 20_000);
+});
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index 9ad1d964e8..4715ef3c16 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -157,6 +157,7 @@ export const traceBatchQueueProcessor: Processor<
const queryOptions = {
maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,
maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,
+ requestTimeoutMs: env.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS,
experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
queryId: randomUUID(),
};
@@ -205,20 +206,57 @@ export const traceBatchQueueProcessor: Processor<
});
}
+ // ClickHouse sockets time out when idle, so the reader never waits on
+ // transcript or Topics work unless queued payloads exceed the cap.
+ // Traces are still summarized one at a time per batch to keep at most
+ // one pending request in the tokenizer pool shared with ingestion.
+ const pendingSummaries: Promise<void>[] = [];
+ let pendingSummaryBytes = 0;
+ let summaryFailed = false;
+ let summaryError: unknown;
+ const enqueueSummary = (observations: Observation[], bytes: number) => {
+ pendingSummaryBytes += bytes;
+ const enqueuedAt = performance.now();
+ const summary = (pendingSummaries.at(-1) ?? Promise.resolve())
+ .then(async () => {
+ // Let socket reads run between traces' synchronous assembly.
+ await new Promise((resolve) => setImmediate(resolve));
+ recordDistribution(
+ "langfuse.trace_batch.summary_queue_wait_ms",
+ performance.now() - enqueuedAt,
+ );
+ if (!summaryFailed)
+ topicsOutcomes.push(await summarizeTraceBatch(observations));
+ })
+ .catch((error: unknown) => {
+ if (!summaryFailed) summaryError = error;
+ summaryFailed = true;
+ })
+ .finally(() => {
+ pendingSummaryBytes -= bytes;
+ pendingSummaries.shift();
+ });
+ pendingSummaries.push(summary);
+ };
activeReads++;
- let pendingTokenization: Promise<void> | undefined;
try {
recordTraceBatchActiveReads();
let traceObservations: Observation[] = [];
+ let traceBytes = 0;
for await (const event of getTraceBatchEventStream(batch, queryOptions)) {
observationCount++;
foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));
foundProjects.add(event.project_id);
// Logical UTF-8 payload size, excluding JSON transport and compression.
- inputBytes += Buffer.byteLength(event.input);
- outputBytes += Buffer.byteLength(event.output);
+ const eventInputBytes = Buffer.byteLength(event.input);
+ const eventOutputBytes = Buffer.byteLength(event.output);
+ let eventBytes = eventInputBytes + eventOutputBytes;
+ inputBytes += eventInputBytes;
+ outputBytes += eventOutputBytes;
for (const [key, value] of Object.entries(event.metadata)) {
- metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);
+ const bytes = Buffer.byteLength(key) + Buffer.byteLength(value);
+ metadataBytes += bytes;
+ eventBytes += bytes;
}
const previous = traceObservations[0];
if (
@@ -226,16 +264,25 @@ export const traceBatchQueueProcessor: Processor<
(previous.projectId !== event.project_id ||
previous.traceId !== event.trace_id)
) {
- // Overlap tokenization with reading the next trace, but allow only
- // one pending estimate per batch so queued payloads stay bounded.
- await pendingTokenization;
- pendingTokenization = summarizeTraceBatch(traceObservations).then(
- (result) => {
- topicsOutcomes.push(result);
- },
- );
+ if (summaryFailed) break;
+ enqueueSummary(traceObservations, traceBytes);
traceObservations = [];
+ traceBytes = 0;
+ if (
+ pendingSummaryBytes >
+ env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES
+ ) {
+ recordIncrement("langfuse.trace_batch.summary_backpressure", 1);
+ // Resume as soon as completed work brings the queue below the cap.
+ while (
+ pendingSummaryBytes >
+ env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES
+ )
+ await pendingSummaries[0];
+ }
+ if (summaryFailed) break;
}
+ traceBytes += eventBytes;
traceObservations.push(
convertObservation({
...event,
@@ -253,32 +300,18 @@ export const traceBatchQueueProcessor: Processor<
);
}
// Reaching EOF completes the last trace; a failed stream must not flush it.
- if (traceObservations.length) {
- await pendingTokenization;
- pendingTokenization = summarizeTraceBatch(traceObservations).then(
- (result) => {
- topicsOutcomes.push(result);
- },
- );
- }
- } catch (error) {
- // Only rows consumed before the failure; never count these as successful throughput.
- for (const [name, value] of [
- ["observation_count", observationCount],
- ["input_bytes", inputBytes],
- ] as const) {
- recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);
- }
- throw error;
+ if (traceObservations.length && !summaryFailed)
+ enqueueSummary(traceObservations, traceBytes);
} finally {
try {
- // Drain accepted tokenization promises even when the stream fails.
- await pendingTokenization;
+ // Drain accepted summaries even when the stream fails.
+ await pendingSummaries.at(-1);
} finally {
activeReads--;
recordTraceBatchActiveReads();
}
}
+ if (summaryFailed) throw summaryError;
recordDistribution(
"langfuse.trace_batch.observation_count",
@@ -310,6 +343,15 @@ export const traceBatchQueueProcessor: Processor<
ioMetadataBytes,
};
} finally {
+ if (queryId && outcome === "failure") {
+ // Consumed rows from failed attempts are not successful throughput.
+ for (const [name, value] of [
+ ["observation_count", observationCount],
+ ["input_bytes", inputBytes],
+ ] as const) {
+ recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);
+ }
+ }
const durationMs = performance.now() - startedAt;
span?.setAttributes({
"langfuse.trace_batch.outcome": outcome,