Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): optimize and configure trace batch reads (#17497)

Lysander Kiesel1de31c0e40ee
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index 37bd3f3d90..1e09792f5b 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -26,7 +26,9 @@
- Internal trace-batch queue: `src/server/redis/traceBatch.ts` (cloud-gated);
payloads in `src/server/queues.ts` accept persisted single-project jobs.
Full-event streaming reads live in `src/server/repositories/trace-batch.ts`;
- the worker owns experiment enablement and lifecycle.
+ the worker owns experiment enablement and lifecycle. Reader options control
+ per-query threads/block size and experiment attribution; retain exact tenant
+ pairs and per-trace time windows when changing parameter chunking.
- Code evaluator dispatcher/error contract: `src/server/evals/codeEvalDispatcherTypes.ts`. Keep provider mappings, user-visible messages, and worker terminal-outcome classification aligned when adding an error code.
- Dashboard/monitor query feature (data model + server-only builder/executor): `src/features/query/*`
- Query-builder AST (server half, WIP): `src/server/query-ast/*` — golden-SQL
diff --git a/packages/shared/src/server/clickhouse/client.test.ts b/packages/shared/src/server/clickhouse/client.test.ts
index d7332c699d..0df53c7fc1 100644
--- a/packages/shared/src/server/clickhouse/client.test.ts
+++ b/packages/shared/src/server/clickhouse/client.test.ts
@@ -115,6 +115,17 @@ describe("ClickHouseClientManager compatibility settings", () => {
});
});
+ it("uses a new cached client when response compression is enabled", () => {
+ clickhouseClient();
+ clickhouseClient({ compression: { response: true } });
+
+ expect(mocks.createClient).toHaveBeenCalledTimes(2);
+ expect(mocks.createClient.mock.calls[0][0].compression).toBeUndefined();
+ expect(mocks.createClient.mock.calls[1][0].compression).toEqual({
+ response: true,
+ });
+ });
+
it("lets explicit client settings override derived timeout settings", () => {
clickhouseClient({
request_timeout: 120_000,
diff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.ts
index 43b4a6e112..dc5f19ad3e 100644
--- a/packages/shared/src/server/clickhouse/client.ts
+++ b/packages/shared/src/server/clickhouse/client.ts
@@ -100,6 +100,7 @@ export class ClickHouseClientManager {
...(opts.request_timeout
? { request_timeout: opts.request_timeout }
: {}),
+ ...(opts.compression ? { compression: opts.compression } : {}),
// Include any other relevant config options
};
diff --git a/packages/shared/src/server/clickhouse/queryTags.test.ts b/packages/shared/src/server/clickhouse/queryTags.test.ts
index fa13bf4df1..70da06656d 100644
--- a/packages/shared/src/server/clickhouse/queryTags.test.ts
+++ b/packages/shared/src/server/clickhouse/queryTags.test.ts
@@ -1,7 +1,48 @@
import { describe, expect, it } from "vitest";
-import { buildClickHouseLogComment } from "./queryTags";
+import { context, propagation } from "@opentelemetry/api";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import {
+ buildClickHouseLogComment,
+ CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS,
+} from "./queryTags";
describe("ClickHouse query tags", () => {
+ it("adds experiment attribution without replacing entrypoint or project attribution", () => {
+ context.setGlobalContextManager(
+ new AsyncLocalStorageContextManager().enable(),
+ );
+ const entrypoint = propagation.setBaggage(
+ context.active(),
+ propagation.createBaggage({
+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface]: { value: "worker" },
+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.route]: {
+ value: "langfuse.queue.trace_batch",
+ },
+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.projectId]: {
+ value: "producer-project",
+ },
+ }),
+ );
+ try {
+ const tags = context.with(entrypoint, () =>
+ JSON.parse(
+ buildClickHouseLogComment({
+ projectId: "MULTI_PROJECT",
+ experimentId: "arm-b",
+ }),
+ ),
+ );
+ expect(tags).toEqual({
+ tag_schema_version: "1",
+ surface: "worker",
+ route: "langfuse.queue.trace_batch",
+ projectId: "MULTI_PROJECT",
+ experimentId: "arm-b",
+ });
+ } finally {
+ context.disable();
+ }
+ });
it("builds v1 log comments from entrypoint context", () => {
const logComment = buildClickHouseLogComment({
surface: "publicapi",
diff --git a/packages/shared/src/server/clickhouse/queryTags.ts b/packages/shared/src/server/clickhouse/queryTags.ts
index 1907205a2b..50ebfd0469 100644
--- a/packages/shared/src/server/clickhouse/queryTags.ts
+++ b/packages/shared/src/server/clickhouse/queryTags.ts
@@ -14,6 +14,7 @@ export const clickHouseQuerySurfaces = [
export type ClickHouseQuerySurface = (typeof clickHouseQuerySurfaces)[number];
export type ClickHouseQueryTags = {
+ experimentId?: string;
surface?: ClickHouseQuerySurface | (string & {});
route?: string;
projectId?: string;
@@ -23,6 +24,7 @@ export type ClickHouseQueryTags = {
};
export type NormalizedClickHouseQueryTags = {
+ experimentId?: string;
tag_schema_version: typeof CLICKHOUSE_QUERY_TAG_SCHEMA_VERSION;
surface: ClickHouseQuerySurface | typeof UNKNOWN_CLICKHOUSE_QUERY_TAG_VALUE;
route?: string;
@@ -87,6 +89,9 @@ export function normalizeClickHouseQueryTags(
: UNKNOWN_CLICKHOUSE_QUERY_TAG_VALUE,
...(normalizedRoute ? { route: normalizedRoute } : {}),
...(projectId ? { projectId } : {}),
+ ...(providedTags?.experimentId
+ ? { experimentId: providedTags.experimentId }
+ : {}),
...(isPublicApi && normalizedSdkName ? { sdkName: normalizedSdkName } : {}),
...(isPublicApi && normalizedSdkVersion
? { sdkVersion: normalizedSdkVersion }
diff --git a/packages/shared/src/server/repositories/trace-batch.test.ts b/packages/shared/src/server/repositories/trace-batch.test.ts
new file mode 100644
index 0000000000..db8cd0b39a
--- /dev/null
+++ b/packages/shared/src/server/repositories/trace-batch.test.ts
@@ -0,0 +1,108 @@
+import { randomUUID } from "node:crypto";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { getTraceBatchEventStream } from "./trace-batch";
+import { queryClickhouseStream } from "./clickhouse";
+
+// Exercise the real query builder, intercepting execution before any client loads.
+vi.mock("./clickhouse", async () => ({
+ TupleParam: (await import("@clickhouse/client")).TupleParam,
+ queryClickhouseStream: vi.fn(),
+}));
+vi.mock("./index", () => ({ clickhouseCompliantRandomCharacters: vi.fn() }));
+vi.mock("../clickhouse/client", () => ({
+ convertDateToClickhouseDateTime: vi.fn(),
+}));
+
+afterEach(() => vi.clearAllMocks());
+
+describe("trace batch query controls", () => {
+ const traces = Array.from({ length: 2_001 }, (_, index) => ({
+ projectId: index % 2 ? "one" : "two",
+ traceId: `trace-${index}`,
+ minStart: 1_000 + index,
+ maxStart: 2_000 + index,
+ }));
+
+ it("bounds 10,000-trace HTTP parameters for distinct projects and identical windows", async () => {
+ vi.mocked(queryClickhouseStream).mockImplementation(async function* () {});
+ for (const sharedWindow of [false, true]) {
+ const projectId = randomUUID();
+ const members = Array.from({ length: 10_000 }, (_, index) => ({
+ projectId: sharedWindow ? projectId : randomUUID(),
+ traceId: randomUUID(),
+ minStart: 1_000 + (sharedWindow ? 0 : index),
+ maxStart: 2_000 + (sharedWindow ? 0 : index),
+ }));
+ for await (const _ of getTraceBatchEventStream({ traces: members })) {
+ throw new Error("Unexpected row");
+ }
+ const sent = vi.mocked(queryClickhouseStream).mock.lastCall![0];
+ expect(sent.useMultipartParamsAuto).toBe(true);
+ expect(sent.clickhouseConfigs).toEqual({
+ compression: { response: true },
+ });
+ expect(Buffer.byteLength(sent.query)).toBeLessThan(262_144);
+ expect(Object.keys(sent.params ?? {}).length).toBeLessThan(1_000);
+ for (const value of Object.values(sent.params ?? {})) {
+ // UUID strings require no transport escaping; JSON conservatively includes
+ // TupleParam property names that ClickHouse's tuple encoding omits.
+ expect(Buffer.byteLength(JSON.stringify(value))).toBeLessThan(131_072);
+ }
+ const projectLists = Object.entries(sent.params ?? {})
+ .filter(([name]) => /^projectIds/.test(name))
+ .flatMap(([, ids]) => ids as string[]);
+ expect(new Set(projectLists)).toEqual(
+ new Set(members.map((t) => t.projectId)),
+ );
+ const timeTraceLists = Object.entries(sent.params ?? {})
+ .filter(([name]) => /^g\d+t$/.test(name))
+ .flatMap(([, ids]) => (ids as string[][]).flat());
+ expect(new Set(timeTraceLists)).toEqual(
+ new Set(members.map((t) => t.traceId)),
+ );
+ }
+ });
+
+ it("passes controls through chunked and single-project reads without changing membership or buffering rows", async () => {
+ const row = { project_id: "one", trace_id: "trace-1" };
+ const failure = new Error("stream failed");
+ let requestedSecondRow = false;
+ vi.mocked(queryClickhouseStream).mockImplementation(async function* () {
+ yield row;
+ requestedSecondRow = true;
+ throw failure;
+ });
+
+ for (const members of [traces.slice(1, 2), traces]) {
+ const baselineStream = getTraceBatchEventStream({ traces: members });
+ await baselineStream.next();
+ const baseline = vi.mocked(queryClickhouseStream).mock.lastCall![0];
+ await baselineStream.return(undefined);
+ for (const maxBlockSize of [undefined, 512]) {
+ requestedSecondRow = false;
+ const stream = getTraceBatchEventStream(
+ { traces: members },
+ { maxThreads: 1, maxBlockSize, experimentId: "arm-b" },
+ );
+ expect(await stream.next()).toEqual({ value: row, done: false });
+ expect(requestedSecondRow).toBe(false);
+ const sent = vi.mocked(queryClickhouseStream).mock.lastCall![0];
+ expect(sent).toEqual({
+ ...baseline,
+ tags: { ...baseline.tags, experimentId: "arm-b" },
+ clickhouseSettings: {
+ ...baseline.clickhouseSettings,
+ max_threads: 1,
+ ...(maxBlockSize === undefined
+ ? {}
+ : { max_block_size: String(maxBlockSize) }),
+ },
+ });
+ if (maxBlockSize === undefined) {
+ expect(sent.clickhouseSettings).not.toHaveProperty("max_block_size");
+ }
+ await expect(stream.next()).rejects.toBe(failure);
+ }
+ }
+ });
+});
diff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.ts
index 74fbf53e74..28c0b862f2 100644
--- a/packages/shared/src/server/repositories/trace-batch.ts
+++ b/packages/shared/src/server/repositories/trace-batch.ts
@@ -22,24 +22,116 @@ type TraceBatchEventRow = {
};
const TRACE_QUERY_BUFFER_MS = 2 * 60_000;
+// Chunk sizes keep a 10,000-trace job with UUID-sized IDs under ClickHouse's
+// default 1,000 HTTP fields and 128 KiB per field. Unrolling one predicate per
+// window would also exceed default max_query_size.
+const TIME_GROUP_PARAM_CHUNK_SIZE = 50;
+const TRACE_ID_PARAM_CHUNK_SIZE = 2_000;
+const TRACE_PAIR_PARAM_CHUNK_SIZE = 1_000;
-/**
- * Stream full events_full payloads into the worker without retaining a whole batch.
- * Rows reflect the events table's current merge state, as in other event reads;
- * this read-only experiment does not deduplicate versions or enrich model data.
- */
-export async function* getTraceBatchEventStream(props: {
+const chunkItems = <T>(items: readonly T[], size: number) => {
+ const chunks: T[][] = [];
+ for (let offset = 0; offset < items.length; offset += size) {
+ chunks.push(items.slice(offset, offset + size));
+ }
+ return chunks;
+};
+
+type TraceBatchEventStreamProps = {
traces: ReadonlyArray<{
projectId: string;
traceId: string;
minStart: number;
maxStart: number;
}>;
-}): AsyncGenerator<TraceBatchEventRow> {
- if (props.traces.length === 0) return;
+};
+const buildTraceBatchEventQuery = (props: TraceBatchEventStreamProps) => {
+ // Bound the total IDs before grouping: one popular time window may contain
+ // the entire job, and still must fit into a single HTTP parameter field.
+ const timeGroupChunks = chunkItems(
+ props.traces,
+ TRACE_ID_PARAM_CHUNK_SIZE,
+ ).flatMap((traceChunk) => {
+ const bufferedGroups = new Map<
+ string,
+ {
+ projectId: string;
+ minStart: number;
+ maxStart: number;
+ traceIds: Set<string>;
+ }
+ >();
+ for (const trace of traceChunk) {
+ const minStart = trace.minStart - TRACE_QUERY_BUFFER_MS;
+ const maxStart = trace.maxStart + TRACE_QUERY_BUFFER_MS;
+ const key = JSON.stringify([trace.projectId, minStart, maxStart]);
+ const group = bufferedGroups.get(key);
+ if (group) {
+ group.traceIds.add(trace.traceId);
+ } else {
+ bufferedGroups.set(key, {
+ projectId: trace.projectId,
+ minStart,
+ maxStart,
+ traceIds: new Set([trace.traceId]),
+ });
+ }
+ }
+ return chunkItems(
+ [...bufferedGroups.values()],
+ TIME_GROUP_PARAM_CHUNK_SIZE,
+ );
+ });
+ const timeGroupParams: Record<string, unknown> = {};
+ const timeGroupPredicates: string[] = [];
+ for (const [chunkIndex, chunk] of timeGroupChunks.entries()) {
+ const projectParam = `g${chunkIndex}p`;
+ const tracesParam = `g${chunkIndex}t`;
+ const minParam = `g${chunkIndex}l`;
+ const maxParam = `g${chunkIndex}u`;
+ timeGroupParams[projectParam] = chunk.map(({ projectId }) => projectId);
+ timeGroupParams[tracesParam] = chunk.map(({ traceIds }) => [...traceIds]);
+ timeGroupParams[minParam] = chunk.map(({ minStart }) => minStart);
+ timeGroupParams[maxParam] = chunk.map(({ maxStart }) => maxStart);
+ timeGroupPredicates.push(
+ `arrayExists((group_project_id, group_trace_ids, min_ms, max_ms) -> e.project_id = group_project_id AND has(group_trace_ids, e.trace_id) AND e.start_time BETWEEN fromUnixTimestamp64Milli(min_ms) AND fromUnixTimestamp64Milli(max_ms), {${projectParam}:Array(String)}, {${tracesParam}:Array(Array(String))}, {${minParam}:Array(Int64)}, {${maxParam}:Array(Int64)})`,
+ );
+ }
const projectIds = [...new Set(props.traces.map((trace) => trace.projectId))];
const traceIds = [...new Set(props.traces.map((trace) => trace.traceId))];
+ const projectParams: Record<string, unknown> = {};
+ const projectPredicates = chunkItems(
+ projectIds,
+ TRACE_ID_PARAM_CHUNK_SIZE,
+ ).map((chunk, index) => {
+ const name = `projectIds${index}`;
+ projectParams[name] = chunk;
+ return `e.project_id IN ({${name}: Array(String)})`;
+ });
+ const traceIdParams: Record<string, unknown> = {};
+ const traceIdPredicates = chunkItems(traceIds, TRACE_ID_PARAM_CHUNK_SIZE).map(
+ (chunk, index) => {
+ const name = `traceIds${index}`;
+ traceIdParams[name] = chunk;
+ return `e.trace_id IN ({${name}: Array(String)})`;
+ },
+ );
+ const hashPredicates = Object.keys(traceIdParams).map(
+ (name) =>
+ `xxHash32(e.trace_id) IN (SELECT arrayJoin(arrayMap(id -> xxHash32(id), {${name}: Array(String)})))`,
+ );
+ const pairParams: Record<string, unknown> = {};
+ const pairPredicates = chunkItems(
+ props.traces,
+ TRACE_PAIR_PARAM_CHUNK_SIZE,
+ ).map((chunk, index) => {
+ const name = `tracePairs${index}`;
+ pairParams[name] = chunk.map(
+ ({ projectId, traceId }) => new TupleParam([projectId, traceId]),
+ );
+ return `(e.project_id, e.trace_id) IN {${name}: Array(Tuple(String, String))}`;
+ });
const builder = new EventsQueryBuilder({ projectId: NoProjectId })
.selectRaw(
"e.project_id",
@@ -58,49 +150,75 @@ export async function* getTraceBatchEventStream(props: {
// Read events_full rather than the smaller, truncated events_core table.
.forceFullTable()
// Separate filters retain project-prefix and trace-index pruning.
- .whereRaw("e.project_id IN ({projectIds: Array(String)})", { projectIds })
- .whereRaw("e.trace_id IN ({traceIds: Array(String)})", { traceIds })
+ .whereRaw(`(${projectPredicates.join(" OR ")})`, projectParams)
+ .whereRaw(`(${traceIdPredicates.join(" OR ")})`, traceIdParams)
// Independent IN lists also match crossed pairs. Only these exact tenant/
// trace pairs may return payloads, even when projects share trace IDs.
- .whereRaw(
- "(e.project_id, e.trace_id) IN {tracePairs: Array(Tuple(String, String))}",
- {
- tracePairs: props.traces.map(
- ({ projectId, traceId }) => new TupleParam([projectId, traceId]),
- ),
- },
- )
+ .whereRaw(`(${pairPredicates.join(" OR ")})`, pairParams)
// Equality filters add this primary-key hash condition automatically; IN
// needs it explicitly. The exact IDs above also exclude hash collisions.
- .whereRaw(
- "xxHash32(e.trace_id) IN (SELECT arrayJoin(arrayMap(id -> xxHash32(id), {traceIds: Array(String)})))",
- )
- // One shared window enables partition/granule pruning. It can include
- // observations beyond a selected trace's own bounds when another trace
- // widens the batch window; per-trace predicates can narrow that coverage.
+ .whereRaw(`(${hashPredicates.join(" OR ")})`)
+ // Keep one shared outer window for partition/granule pruning.
.whereRaw(
"e.start_time >= fromUnixTimestamp64Milli({batchMinStart: Int64}) AND e.start_time <= fromUnixTimestamp64Milli({batchMaxStart: Int64})",
{
batchMinStart:
- Math.min(...props.traces.map((trace) => trace.minStart)) -
+ Math.min(...props.traces.map(({ minStart }) => minStart)) -
TRACE_QUERY_BUFFER_MS,
batchMaxStart:
- Math.max(...props.traces.map((trace) => trace.maxStart)) +
+ Math.max(...props.traces.map(({ maxStart }) => maxStart)) +
TRACE_QUERY_BUFFER_MS,
},
+ )
+ // Batch companions must not widen another trace's required coverage.
+ // Identical project/time windows share one ID list; repeated pair intervals
+ // remain separate OR branches and therefore return each matching row once.
+ .whereRaw(
+ `(${timeGroupPredicates.join(`
+ OR `)})`,
+ timeGroupParams,
);
- const { query, params } = builder.buildWithParams();
+ return {
+ ...builder.buildWithParams(),
+ projectIds,
+ };
+};
+
+/**
+ * Stream full events_full payloads into the worker without retaining a whole batch.
+ * Rows reflect the events table's current merge state, as in other event reads;
+ * this read-only experiment does not deduplicate versions or enrich model data.
+ */
+export async function* getTraceBatchEventStream(
+ props: TraceBatchEventStreamProps,
+ options: {
+ maxThreads?: number;
+ maxBlockSize?: number;
+ experimentId?: string;
+ } = {},
+): AsyncGenerator<TraceBatchEventRow> {
+ if (props.traces.length === 0) return;
+
+ const { query, params, projectIds } = buildTraceBatchEventQuery(props);
yield* queryClickhouseStream<TraceBatchEventRow>({
query,
params,
+ useMultipartParamsAuto: true,
tags: {
projectId: projectIds.length === 1 ? projectIds[0] : "MULTI_PROJECT",
+ ...(options.experimentId ? { experimentId: options.experimentId } : {}),
},
preferredClickhouseService: "EventsReadOnly",
+ clickhouseConfigs: {
+ compression: { response: true },
+ },
// Bound background-read CPU/time; timeouts fail instead of returning partial results.
clickhouseSettings: {
- max_threads: 2,
+ max_threads: options.maxThreads ?? 2,
+ ...(options.maxBlockSize === undefined
+ ? {}
+ : { max_block_size: String(options.maxBlockSize) }),
max_execution_time: 30,
timeout_overflow_mode: "throw",
},
diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.ts
index 564c57d035..6dcb82efce 100644
--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts
@@ -65,7 +65,7 @@ const findFilterOption = (
) => rows.find((row) => row.column === column && row.value === value);
describe("Clickhouse Events Repository Test", () => {
- it("streams complete trace batches with full payloads, exact tenant/trace pairs and shared batch time bounds", async () => {
+ it("streams complete trace batches with full payloads and exact per-trace time bounds", async () => {
const batchProjectId = randomUUID();
const otherProjectId = randomUUID();
const firstTraceId = randomUUID();
@@ -93,26 +93,44 @@ describe("Clickhouse Events Repository Test", () => {
trace_id: secondTraceId,
start_time: new Date(start + 600_000 + buffer),
});
- // Another trace can widen the shared window beyond this trace's own bounds.
- const insideBatchWindow = createEvent({
+ const longTraceUpperEndpoint = createEvent({
+ project_id: batchProjectId,
+ trace_id: secondTraceId,
+ start_time: new Date(start + 1_800_000 + buffer),
+ });
+ const companionOnlyRow = createEvent({
project_id: batchProjectId,
trace_id: firstTraceId,
start_time: new Date(start + buffer + 1),
});
+ const repeatedIntervalRow = createEvent({
+ project_id: batchProjectId,
+ trace_id: firstTraceId,
+ start_time: new Date(start + 1_200_000),
+ });
+ const otherProjectWindowRow = createEvent({
+ project_id: otherProjectId,
+ trace_id: firstTraceId,
+ start_time: new Date(start),
+ });
await createEventsCh([
first,
second,
- insideBatchWindow,
+ longTraceUpperEndpoint,
+ companionOnlyRow,
+ repeatedIntervalRow,
+ // This row only matches the same trace ID's window in another project.
+ otherProjectWindowRow,
createEvent({
project_id: otherProjectId,
trace_id: otherTraceId,
start_time: new Date(start),
}),
- // The same trace ID in two projects represents two different traces.
+ // The same trace ID in two projects has independent time windows.
createEvent({
project_id: otherProjectId,
trace_id: firstTraceId,
- start_time: new Date(start),
+ start_time: new Date(start + 600_000),
}),
// These crossed pairs match both independent IN lists, but are unrequested.
createEvent({
@@ -143,7 +161,7 @@ describe("Clickhouse Events Repository Test", () => {
createEvent({
project_id: batchProjectId,
trace_id: secondTraceId,
- start_time: new Date(start + 600_000 + buffer + 1),
+ start_time: new Date(start + 1_800_000 + buffer + 1),
}),
]);
@@ -169,7 +187,9 @@ describe("Clickhouse Events Repository Test", () => {
let rowCount = 0;
let fullPayloadSeen = false;
- let insideBatchWindowSeen = false;
+ let companionOnlyRowSeen = false;
+ let otherProjectWindowRowSeen = false;
+ let repeatedIntervalRowCount = 0;
const foundTraces = new Set<string>();
for await (const event of getTraceBatchEventStream({
traces: [
@@ -183,7 +203,7 @@ describe("Clickhouse Events Repository Test", () => {
projectId: batchProjectId,
traceId: secondTraceId,
minStart: start + 600_000,
- maxStart: start + 600_000,
+ maxStart: start + 1_800_000,
},
{
projectId: otherProjectId,
@@ -194,12 +214,25 @@ describe("Clickhouse Events Repository Test", () => {
{
projectId: otherProjectId,
traceId: firstTraceId,
+ minStart: start + 600_000,
+ maxStart: start + 600_000,
+ },
+ // Repeated identical intervals must not duplicate output rows.
+ {
+ projectId: batchProjectId,
+ traceId: firstTraceId,
minStart: start,
maxStart: start,
},
- // Exercise the maximum configurable batch size. Missing IDs stress
- // HTTP parameters and query size without unrelated fixture rows.
- ...Array.from({ length: 996 }, () => ({
+ // A repeated pair may contribute a separate required interval.
+ {
+ projectId: batchProjectId,
+ traceId: firstTraceId,
+ minStart: start + 1_200_000,
+ maxStart: start + 1_200_000,
+ },
+ // Missing IDs stress HTTP parameters without unrelated fixture rows.
+ ...Array.from({ length: 994 }, () => ({
projectId: batchProjectId,
traceId: randomUUID(),
minStart: start,
@@ -209,8 +242,14 @@ describe("Clickhouse Events Repository Test", () => {
})) {
rowCount++;
foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));
- if (event.span_id === insideBatchWindow.span_id) {
- insideBatchWindowSeen = true;
+ if (event.span_id === companionOnlyRow.span_id) {
+ companionOnlyRowSeen = true;
+ }
+ if (event.span_id === otherProjectWindowRow.span_id) {
+ otherProjectWindowRowSeen = true;
+ }
+ if (event.span_id === repeatedIntervalRow.span_id) {
+ repeatedIntervalRowCount++;
}
if (event.span_id === first.span_id) {
fullPayloadSeen = true;
@@ -225,7 +264,9 @@ describe("Clickhouse Events Repository Test", () => {
}
}
expect(fullPayloadSeen).toBe(true);
- expect(insideBatchWindowSeen).toBe(true);
+ expect(companionOnlyRowSeen).toBe(false);
+ expect(otherProjectWindowRowSeen).toBe(false);
+ expect(repeatedIntervalRowCount).toBe(1);
expect(foundTraces).toEqual(
new Set([
JSON.stringify([batchProjectId, firstTraceId]),
@@ -234,9 +275,39 @@ describe("Clickhouse Events Repository Test", () => {
JSON.stringify([otherProjectId, firstTraceId]),
]),
);
- expect(rowCount).toBe(extraRowCount + 5);
+ expect(rowCount).toBe(extraRowCount + 6);
}, 60_000);
+ it("returns matching rows across parameter chunks in a 10,000-trace batch", async () => {
+ const start = Date.now();
+ const traces = Array.from({ length: 10_000 }, (_, index) => ({
+ projectId: randomUUID(),
+ traceId: randomUUID(),
+ minStart: start + index,
+ maxStart: start + index,
+ }));
+ // Exercise both sides of time-group, pair, and ID chunk boundaries, plus
+ // the final chunk. Distinct projects also exercise project parameter chunks.
+ const expectedRows = [0, 49, 50, 999, 1_000, 1_999, 2_000, 9_999].map(
+ (index) =>
+ createEvent({
+ project_id: traces[index].projectId,
+ trace_id: traces[index].traceId,
+ start_time: new Date(traces[index].minStart),
+ }),
+ );
+ await createEventsCh(expectedRows);
+
+ const foundSpanIds: string[] = [];
+ for await (const event of getTraceBatchEventStream({ traces })) {
+ foundSpanIds.push(event.span_id);
+ }
+
+ expect(foundSpanIds.sort()).toEqual(
+ expectedRows.map(({ span_id }) => span_id).sort(),
+ );
+ }, 120_000);
+
it("should kill redis connection", () => {
// we need at least one test case to avoid hanging
// redis connection when everything else is skipped.
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index 5fdbfd51a7..c0310a725f 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -20,7 +20,8 @@
`src/queues/traceBatchQueue.ts`; controls and Redis lifecycle are documented in
`src/features/traceBatching/README.md`. Keep producer, dispatcher, consumer and reads
independently default-off and cloud-gated. Do not expose these PoC controls in
- local or production env templates.
+ local or production env templates. Reader query controls are independent of
+ locality selection; logs must preserve separate input/output/metadata metrics.
- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.
- Service layer: `src/services/*`
- Rust addon (`@langfuse/native`): telemetry init and the startup hello call live
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 5bd76cfa9a..482d20dd59 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -126,11 +126,27 @@ const EnvSchema = z.object({
.int()
.positive()
.default(2),
+ LANGFUSE_TRACE_BATCH_MAX_THREADS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(2),
+ LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: z.coerce
+ .number()
+ .int()
+ .positive()
+ .optional(),
+ LANGFUSE_TRACE_BATCH_EXPERIMENT_ID: z
+ .string()
+ .min(1)
+ .max(64)
+ .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/)
+ .optional(),
LANGFUSE_TRACE_BATCH_MAX_SIZE: z.coerce
.number()
.int()
.positive()
- .max(1_000)
+ .max(10_000)
.default(60),
LANGFUSE_TRACE_BATCH_STRATEGY: z
.enum(["project", "locality"])
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index 5462ff206e..1f4f235732 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -7,19 +7,22 @@ changes. These internal controls are intentionally absent from env templates.
## Controls
-| Variable | Default | Role |
-| --------------------------------------------- | --------- | ------------------------------------------------------------- |
-| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
-| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
-| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
-| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
-| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |
-| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
-| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 1,000. |
-| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |
-| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
-| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
+| Variable | Default | Role |
+| --------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------- |
+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |
+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |
+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |
+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |
+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
Producer, dispatcher and consumer can run independently. Sampling reuses the
evaluator's deterministic trace-ID sampler; every observation of the same trace
@@ -63,13 +66,45 @@ Sizes count UTF-8 bytes, including metadata keys and values, excluding JSON
transport framing/escaping and compression. They do not measure ClickHouse bytes
scanned or per-trace ingestion size; retries can record another batch sample.
Exact `(projectId, traceId)`
-pairs protect tenant boundaries; project, trace hash and a shared batch time
-window prune reads. `TRACE_QUERY_BUFFER_MS` pads the earliest and latest recorded
-start times by two minutes each. This fixed query margin can include observations
-outside the recorded bounds; it neither waits for arrivals nor guarantees trace
-completeness. Query limits are fixed at two threads and 30 seconds with
-timeouts failing the job. Per-trace windows, query tuning controls and allocation
-based on trace size are separate experiments.
+pairs protect tenant boundaries; project, trace hash and a shared outer time
+window prune reads. Each selected trace also gets its own interval, so unrelated
+batch companions cannot widen the payload returned for that trace.
+`TRACE_QUERY_BUFFER_MS` pads each recorded interval by two minutes on both sides.
+This fixed margin neither waits for arrivals nor guarantees trace completeness;
+repeated intervals for the same pair form a union without duplicating rows.
+Queries use response compression and chunked multipart parameters for up to
+10,000 UUID-sized trace/project IDs. The fixed 30-second timeout fails the job
+rather than accepting partial results. This does not cap observation count,
+query memory or dispatcher snapshot memory.
+
+## Compare reader settings
+
+Establish a fresh control measurement after deploying this reader: both arms
+must use the same per-trace windows and compression. Hold sampling, batch cap,
+concurrency, threads, idle delay and traffic cohort constant; then change only
+block size (and the identifying label):
+
+| Variable | Control | Treatment |
+| ------------------------------------- | --------------------- | ----------------- |
+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | `2` |
+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | `512` |
+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | `trace-block-control` | `trace-block-512` |
+
+Run these settings only in cloud workers with the existing four enablement
+flags explicitly set to `true`. Changing a query setting does not enable any
+part of the pipeline. `max_block_size` is a row-count hint, not a byte or memory
+limit; large inputs may still dominate memory. Keep the initial cap at 120 when
+comparing existing experiments; raising the supported ceiling is not a rollout
+recommendation.
+
+With an experiment ID, each read attempt logs its label, build ID, configured
+threads/block size, batch size, concurrency, sampling and timing controls before
+querying, including failed attempts. The same ID enters ClickHouse
+`log_comment` alongside existing surface/route/project attribution. Leave the ID
+unset to suppress these per-attempt logs. Strategy attribution uses the
+dispatcher metrics described below. Compare
+query memory/latency and worker memory as well as input/output/metadata byte
+metrics. Redis storage load tests do not measure selector or query memory.
### Opt-in locality grouping
@@ -144,5 +179,4 @@ lazy retention rules. Drain and expired jobs are removed immediately on success.
For an already enabled cloud experiment, explicitly set the new read flag to
`true` when deploying this version if reads should continue; leaving it unset
drains instead. Upgrade all producers before relying on native expiry: older
-producers do not install or refresh that TTL. Query controls and load
-test artifacts belong in separate follow-up changes.
+producers do not install or refresh that TTL.
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index cdb29bed53..6edd7b65ec 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { type Job } from "bullmq";
import {
getTraceBatchEventStream,
+ logger,
QueueJobs,
type QueueName,
recordDistribution,
@@ -74,6 +75,84 @@ describe("trace batch queue", () => {
expect(job.opts.removeOnComplete).toBe(true);
});
+ it("forwards overrides and rejects a partially consumed stream without reporting success", async () => {
+ const originalEnv = { ...env };
+ Object.assign(env, {
+ LANGFUSE_TRACE_BATCH_MAX_THREADS: 1,
+ LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: 512,
+ LANGFUSE_TRACE_BATCH_EXPERIMENT_ID: "arm-b",
+ BUILD_ID: "test-build",
+ });
+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
+ const failure = new Error("query failed");
+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
+ yield {
+ project_id: "project",
+ trace_id: "trace",
+ span_id: "span",
+ parent_span_id: null,
+ start_time: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000000",
+ type: "GENERATION",
+ name: "generation",
+ input: "input",
+ output: "output",
+ metadata: {},
+ tool_definitions: {},
+ tool_calls: [],
+ tool_call_names: [],
+ };
+ throw failure;
+ });
+ const payload = {
+ traces: [
+ {
+ projectId: "project",
+ traceId: "trace",
+ minStart: 1_000,
+ maxStart: 2_000,
+ revision: "r",
+ },
+ ],
+ };
+ const job = {
+ data: {
+ id: "batch",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(),
+ payload,
+ },
+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+ try {
+ await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(
+ failure,
+ );
+ expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload, {
+ maxThreads: 1,
+ maxBlockSize: 512,
+ experimentId: "arm-b",
+ });
+ expect(log).toHaveBeenCalledWith(
+ "Trace batch experiment read",
+ expect.objectContaining({
+ experimentId: "arm-b",
+ buildId: "test-build",
+ maxThreads: 1,
+ maxBlockSize: 512,
+ batchTraceCount: 1,
+ }),
+ );
+ expect(recordDistribution).not.toHaveBeenCalled();
+ } finally {
+ env.LANGFUSE_TRACE_BATCH_MAX_THREADS =
+ originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;
+ env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE =
+ originalEnv.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE;
+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID =
+ originalEnv.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;
+ env.BUILD_ID = originalEnv.BUILD_ID;
+ }
+ });
it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {
const ingestionEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;
const dispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
@@ -147,7 +226,10 @@ describe("trace batch queue", () => {
);
}
expect(getTraceBatchEventStream).toHaveBeenCalledTimes(2);
- expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload);
+ expect(getTraceBatchEventStream).toHaveBeenCalledWith(
+ payload,
+ expect.anything(),
+ );
expect(yieldedRows).toBe(8);
for (const [name, bytes] of [
["input_bytes", 20],
@@ -246,9 +328,10 @@ describe("trace batch queue", () => {
"langfuse.trace_batch.found_project_count",
Number(hasRows),
);
- expect(getTraceBatchEventStream).toHaveBeenCalledWith({
- traces: [{ ...trace, projectId: "project" }],
- });
+ expect(getTraceBatchEventStream).toHaveBeenCalledWith(
+ { traces: [{ ...trace, projectId: "project" }] },
+ expect.anything(),
+ );
},
);
});
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index 522b1c4278..834f0b4353 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -1,6 +1,7 @@
import { type Processor } from "bullmq";
import {
getTraceBatchEventStream,
+ logger,
recordDistribution,
recordIncrement,
TraceBatchEventSchema,
@@ -43,7 +44,27 @@ export const traceBatchQueueProcessor: Processor<
let outputBytes = 0;
let metadataBytes = 0;
- for await (const event of getTraceBatchEventStream(batch)) {
+ const queryOptions = {
+ maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,
+ maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,
+ experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
+ };
+ if (queryOptions.experimentId) {
+ logger.info("Trace batch experiment read", {
+ ...queryOptions,
+ // null means no override; the server profile's value is not known here.
+ maxBlockSize: queryOptions.maxBlockSize ?? null,
+ buildId: env.BUILD_ID,
+ batchTraceCount: batch.traces.length,
+ maxBatchSize: env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
+ concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,
+ samplingRate: env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE,
+ idleMs: env.LANGFUSE_TRACE_BATCH_IDLE_MS,
+ dispatchIntervalMs: env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS,
+ });
+ }
+
+ for await (const event of getTraceBatchEventStream(batch, queryOptions)) {
observationCount++;
foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));
foundProjects.add(event.project_id);