🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): optimize and configure trace batch reads (#17497)
Lysander Kiesel1de31c0e40ee
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex 37bd3f3d90..1e09792f5b 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -26,7 +26,9 @@- Internal trace-batch queue: `src/server/redis/traceBatch.ts` (cloud-gated);payloads in `src/server/queues.ts` accept persisted single-project jobs.Full-event streaming reads live in `src/server/repositories/trace-batch.ts`;- the worker owns experiment enablement and lifecycle.+ the worker owns experiment enablement and lifecycle. Reader options control+ per-query threads/block size and experiment attribution; retain exact tenant+ pairs and per-trace time windows when changing parameter chunking.- Code evaluator dispatcher/error contract: `src/server/evals/codeEvalDispatcherTypes.ts`. Keep provider mappings, user-visible messages, and worker terminal-outcome classification aligned when adding an error code.- Dashboard/monitor query feature (data model + server-only builder/executor): `src/features/query/*`- Query-builder AST (server half, WIP): `src/server/query-ast/*` — golden-SQLdiff --git a/packages/shared/src/server/clickhouse/client.test.ts b/packages/shared/src/server/clickhouse/client.test.tsindex d7332c699d..0df53c7fc1 100644--- a/packages/shared/src/server/clickhouse/client.test.ts+++ b/packages/shared/src/server/clickhouse/client.test.ts@@ -115,6 +115,17 @@ describe("ClickHouseClientManager compatibility settings", () => {});});+ it("uses a new cached client when response compression is enabled", () => {+ clickhouseClient();+ clickhouseClient({ compression: { response: true } });++ expect(mocks.createClient).toHaveBeenCalledTimes(2);+ expect(mocks.createClient.mock.calls[0][0].compression).toBeUndefined();+ expect(mocks.createClient.mock.calls[1][0].compression).toEqual({+ response: true,+ });+ });+it("lets explicit client settings override derived timeout settings", () => {clickhouseClient({request_timeout: 120_000,diff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.tsindex 43b4a6e112..dc5f19ad3e 100644--- a/packages/shared/src/server/clickhouse/client.ts+++ b/packages/shared/src/server/clickhouse/client.ts@@ -100,6 +100,7 @@ export class ClickHouseClientManager {...(opts.request_timeout? { request_timeout: opts.request_timeout }: {}),+ ...(opts.compression ? { compression: opts.compression } : {}),// Include any other relevant config options};diff --git a/packages/shared/src/server/clickhouse/queryTags.test.ts b/packages/shared/src/server/clickhouse/queryTags.test.tsindex fa13bf4df1..70da06656d 100644--- a/packages/shared/src/server/clickhouse/queryTags.test.ts+++ b/packages/shared/src/server/clickhouse/queryTags.test.ts@@ -1,7 +1,48 @@import { describe, expect, it } from "vitest";-import { buildClickHouseLogComment } from "./queryTags";+import { context, propagation } from "@opentelemetry/api";+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";+import {+ buildClickHouseLogComment,+ CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS,+} from "./queryTags";describe("ClickHouse query tags", () => {+ it("adds experiment attribution without replacing entrypoint or project attribution", () => {+ context.setGlobalContextManager(+ new AsyncLocalStorageContextManager().enable(),+ );+ const entrypoint = propagation.setBaggage(+ context.active(),+ propagation.createBaggage({+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface]: { value: "worker" },+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.route]: {+ value: "langfuse.queue.trace_batch",+ },+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.projectId]: {+ value: "producer-project",+ },+ }),+ );+ try {+ const tags = context.with(entrypoint, () =>+ JSON.parse(+ buildClickHouseLogComment({+ projectId: "MULTI_PROJECT",+ experimentId: "arm-b",+ }),+ ),+ );+ expect(tags).toEqual({+ tag_schema_version: "1",+ surface: "worker",+ route: "langfuse.queue.trace_batch",+ projectId: "MULTI_PROJECT",+ experimentId: "arm-b",+ });+ } finally {+ context.disable();+ }+ });it("builds v1 log comments from entrypoint context", () => {const logComment = buildClickHouseLogComment({surface: "publicapi",diff --git a/packages/shared/src/server/clickhouse/queryTags.ts b/packages/shared/src/server/clickhouse/queryTags.tsindex 1907205a2b..50ebfd0469 100644--- a/packages/shared/src/server/clickhouse/queryTags.ts+++ b/packages/shared/src/server/clickhouse/queryTags.ts@@ -14,6 +14,7 @@ export const clickHouseQuerySurfaces = [export type ClickHouseQuerySurface = (typeof clickHouseQuerySurfaces)[number];export type ClickHouseQueryTags = {+ experimentId?: string;surface?: ClickHouseQuerySurface | (string & {});route?: string;projectId?: string;@@ -23,6 +24,7 @@ export type ClickHouseQueryTags = {};export type NormalizedClickHouseQueryTags = {+ experimentId?: string;tag_schema_version: typeof CLICKHOUSE_QUERY_TAG_SCHEMA_VERSION;surface: ClickHouseQuerySurface | typeof UNKNOWN_CLICKHOUSE_QUERY_TAG_VALUE;route?: string;@@ -87,6 +89,9 @@ export function normalizeClickHouseQueryTags(: UNKNOWN_CLICKHOUSE_QUERY_TAG_VALUE,...(normalizedRoute ? { route: normalizedRoute } : {}),...(projectId ? { projectId } : {}),+ ...(providedTags?.experimentId+ ? { experimentId: providedTags.experimentId }+ : {}),...(isPublicApi && normalizedSdkName ? { sdkName: normalizedSdkName } : {}),...(isPublicApi && normalizedSdkVersion? { sdkVersion: normalizedSdkVersion }diff --git a/packages/shared/src/server/repositories/trace-batch.test.ts b/packages/shared/src/server/repositories/trace-batch.test.tsnew file mode 100644index 0000000000..db8cd0b39a--- /dev/null+++ b/packages/shared/src/server/repositories/trace-batch.test.ts@@ -0,0 +1,108 @@+import { randomUUID } from "node:crypto";+import { afterEach, describe, expect, it, vi } from "vitest";+import { getTraceBatchEventStream } from "./trace-batch";+import { queryClickhouseStream } from "./clickhouse";++// Exercise the real query builder, intercepting execution before any client loads.+vi.mock("./clickhouse", async () => ({+ TupleParam: (await import("@clickhouse/client")).TupleParam,+ queryClickhouseStream: vi.fn(),+}));+vi.mock("./index", () => ({ clickhouseCompliantRandomCharacters: vi.fn() }));+vi.mock("../clickhouse/client", () => ({+ convertDateToClickhouseDateTime: vi.fn(),+}));++afterEach(() => vi.clearAllMocks());++describe("trace batch query controls", () => {+ const traces = Array.from({ length: 2_001 }, (_, index) => ({+ projectId: index % 2 ? "one" : "two",+ traceId: `trace-${index}`,+ minStart: 1_000 + index,+ maxStart: 2_000 + index,+ }));++ it("bounds 10,000-trace HTTP parameters for distinct projects and identical windows", async () => {+ vi.mocked(queryClickhouseStream).mockImplementation(async function* () {});+ for (const sharedWindow of [false, true]) {+ const projectId = randomUUID();+ const members = Array.from({ length: 10_000 }, (_, index) => ({+ projectId: sharedWindow ? projectId : randomUUID(),+ traceId: randomUUID(),+ minStart: 1_000 + (sharedWindow ? 0 : index),+ maxStart: 2_000 + (sharedWindow ? 0 : index),+ }));+ for await (const _ of getTraceBatchEventStream({ traces: members })) {+ throw new Error("Unexpected row");+ }+ const sent = vi.mocked(queryClickhouseStream).mock.lastCall![0];+ expect(sent.useMultipartParamsAuto).toBe(true);+ expect(sent.clickhouseConfigs).toEqual({+ compression: { response: true },+ });+ expect(Buffer.byteLength(sent.query)).toBeLessThan(262_144);+ expect(Object.keys(sent.params ?? {}).length).toBeLessThan(1_000);+ for (const value of Object.values(sent.params ?? {})) {+ // UUID strings require no transport escaping; JSON conservatively includes+ // TupleParam property names that ClickHouse's tuple encoding omits.+ expect(Buffer.byteLength(JSON.stringify(value))).toBeLessThan(131_072);+ }+ const projectLists = Object.entries(sent.params ?? {})+ .filter(([name]) => /^projectIds/.test(name))+ .flatMap(([, ids]) => ids as string[]);+ expect(new Set(projectLists)).toEqual(+ new Set(members.map((t) => t.projectId)),+ );+ const timeTraceLists = Object.entries(sent.params ?? {})+ .filter(([name]) => /^g\d+t$/.test(name))+ .flatMap(([, ids]) => (ids as string[][]).flat());+ expect(new Set(timeTraceLists)).toEqual(+ new Set(members.map((t) => t.traceId)),+ );+ }+ });++ it("passes controls through chunked and single-project reads without changing membership or buffering rows", async () => {+ const row = { project_id: "one", trace_id: "trace-1" };+ const failure = new Error("stream failed");+ let requestedSecondRow = false;+ vi.mocked(queryClickhouseStream).mockImplementation(async function* () {+ yield row;+ requestedSecondRow = true;+ throw failure;+ });++ for (const members of [traces.slice(1, 2), traces]) {+ const baselineStream = getTraceBatchEventStream({ traces: members });+ await baselineStream.next();+ const baseline = vi.mocked(queryClickhouseStream).mock.lastCall![0];+ await baselineStream.return(undefined);+ for (const maxBlockSize of [undefined, 512]) {+ requestedSecondRow = false;+ const stream = getTraceBatchEventStream(+ { traces: members },+ { maxThreads: 1, maxBlockSize, experimentId: "arm-b" },+ );+ expect(await stream.next()).toEqual({ value: row, done: false });+ expect(requestedSecondRow).toBe(false);+ const sent = vi.mocked(queryClickhouseStream).mock.lastCall![0];+ expect(sent).toEqual({+ ...baseline,+ tags: { ...baseline.tags, experimentId: "arm-b" },+ clickhouseSettings: {+ ...baseline.clickhouseSettings,+ max_threads: 1,+ ...(maxBlockSize === undefined+ ? {}+ : { max_block_size: String(maxBlockSize) }),+ },+ });+ if (maxBlockSize === undefined) {+ expect(sent.clickhouseSettings).not.toHaveProperty("max_block_size");+ }+ await expect(stream.next()).rejects.toBe(failure);+ }+ }+ });+});diff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.tsindex 74fbf53e74..28c0b862f2 100644--- a/packages/shared/src/server/repositories/trace-batch.ts+++ b/packages/shared/src/server/repositories/trace-batch.ts@@ -22,24 +22,116 @@ type TraceBatchEventRow = {};const TRACE_QUERY_BUFFER_MS = 2 * 60_000;+// Chunk sizes keep a 10,000-trace job with UUID-sized IDs under ClickHouse's+// default 1,000 HTTP fields and 128 KiB per field. Unrolling one predicate per+// window would also exceed default max_query_size.+const TIME_GROUP_PARAM_CHUNK_SIZE = 50;+const TRACE_ID_PARAM_CHUNK_SIZE = 2_000;+const TRACE_PAIR_PARAM_CHUNK_SIZE = 1_000;-/**- * Stream full events_full payloads into the worker without retaining a whole batch.- * Rows reflect the events table's current merge state, as in other event reads;- * this read-only experiment does not deduplicate versions or enrich model data.- */-export async function* getTraceBatchEventStream(props: {+const chunkItems = <T>(items: readonly T[], size: number) => {+ const chunks: T[][] = [];+ for (let offset = 0; offset < items.length; offset += size) {+ chunks.push(items.slice(offset, offset + size));+ }+ return chunks;+};++type TraceBatchEventStreamProps = {traces: ReadonlyArray<{projectId: string;traceId: string;minStart: number;maxStart: number;}>;-}): AsyncGenerator<TraceBatchEventRow> {- if (props.traces.length === 0) return;+};+const buildTraceBatchEventQuery = (props: TraceBatchEventStreamProps) => {+ // Bound the total IDs before grouping: one popular time window may contain+ // the entire job, and still must fit into a single HTTP parameter field.+ const timeGroupChunks = chunkItems(+ props.traces,+ TRACE_ID_PARAM_CHUNK_SIZE,+ ).flatMap((traceChunk) => {+ const bufferedGroups = new Map<+ string,+ {+ projectId: string;+ minStart: number;+ maxStart: number;+ traceIds: Set<string>;+ }+ >();+ for (const trace of traceChunk) {+ const minStart = trace.minStart - TRACE_QUERY_BUFFER_MS;+ const maxStart = trace.maxStart + TRACE_QUERY_BUFFER_MS;+ const key = JSON.stringify([trace.projectId, minStart, maxStart]);+ const group = bufferedGroups.get(key);+ if (group) {+ group.traceIds.add(trace.traceId);+ } else {+ bufferedGroups.set(key, {+ projectId: trace.projectId,+ minStart,+ maxStart,+ traceIds: new Set([trace.traceId]),+ });+ }+ }+ return chunkItems(+ [...bufferedGroups.values()],+ TIME_GROUP_PARAM_CHUNK_SIZE,+ );+ });+ const timeGroupParams: Record<string, unknown> = {};+ const timeGroupPredicates: string[] = [];+ for (const [chunkIndex, chunk] of timeGroupChunks.entries()) {+ const projectParam = `g${chunkIndex}p`;+ const tracesParam = `g${chunkIndex}t`;+ const minParam = `g${chunkIndex}l`;+ const maxParam = `g${chunkIndex}u`;+ timeGroupParams[projectParam] = chunk.map(({ projectId }) => projectId);+ timeGroupParams[tracesParam] = chunk.map(({ traceIds }) => [...traceIds]);+ timeGroupParams[minParam] = chunk.map(({ minStart }) => minStart);+ timeGroupParams[maxParam] = chunk.map(({ maxStart }) => maxStart);+ timeGroupPredicates.push(+ `arrayExists((group_project_id, group_trace_ids, min_ms, max_ms) -> e.project_id = group_project_id AND has(group_trace_ids, e.trace_id) AND e.start_time BETWEEN fromUnixTimestamp64Milli(min_ms) AND fromUnixTimestamp64Milli(max_ms), {${projectParam}:Array(String)}, {${tracesParam}:Array(Array(String))}, {${minParam}:Array(Int64)}, {${maxParam}:Array(Int64)})`,+ );+ }const projectIds = [...new Set(props.traces.map((trace) => trace.projectId))];const traceIds = [...new Set(props.traces.map((trace) => trace.traceId))];+ const projectParams: Record<string, unknown> = {};+ const projectPredicates = chunkItems(+ projectIds,+ TRACE_ID_PARAM_CHUNK_SIZE,+ ).map((chunk, index) => {+ const name = `projectIds${index}`;+ projectParams[name] = chunk;+ return `e.project_id IN ({${name}: Array(String)})`;+ });+ const traceIdParams: Record<string, unknown> = {};+ const traceIdPredicates = chunkItems(traceIds, TRACE_ID_PARAM_CHUNK_SIZE).map(+ (chunk, index) => {+ const name = `traceIds${index}`;+ traceIdParams[name] = chunk;+ return `e.trace_id IN ({${name}: Array(String)})`;+ },+ );+ const hashPredicates = Object.keys(traceIdParams).map(+ (name) =>+ `xxHash32(e.trace_id) IN (SELECT arrayJoin(arrayMap(id -> xxHash32(id), {${name}: Array(String)})))`,+ );+ const pairParams: Record<string, unknown> = {};+ const pairPredicates = chunkItems(+ props.traces,+ TRACE_PAIR_PARAM_CHUNK_SIZE,+ ).map((chunk, index) => {+ const name = `tracePairs${index}`;+ pairParams[name] = chunk.map(+ ({ projectId, traceId }) => new TupleParam([projectId, traceId]),+ );+ return `(e.project_id, e.trace_id) IN {${name}: Array(Tuple(String, String))}`;+ });const builder = new EventsQueryBuilder({ projectId: NoProjectId }).selectRaw("e.project_id",@@ -58,49 +150,75 @@ export async function* getTraceBatchEventStream(props: {// Read events_full rather than the smaller, truncated events_core table..forceFullTable()// Separate filters retain project-prefix and trace-index pruning.- .whereRaw("e.project_id IN ({projectIds: Array(String)})", { projectIds })- .whereRaw("e.trace_id IN ({traceIds: Array(String)})", { traceIds })+ .whereRaw(`(${projectPredicates.join(" OR ")})`, projectParams)+ .whereRaw(`(${traceIdPredicates.join(" OR ")})`, traceIdParams)// Independent IN lists also match crossed pairs. Only these exact tenant/// trace pairs may return payloads, even when projects share trace IDs.- .whereRaw(- "(e.project_id, e.trace_id) IN {tracePairs: Array(Tuple(String, String))}",- {- tracePairs: props.traces.map(- ({ projectId, traceId }) => new TupleParam([projectId, traceId]),- ),- },- )+ .whereRaw(`(${pairPredicates.join(" OR ")})`, pairParams)// Equality filters add this primary-key hash condition automatically; IN// needs it explicitly. The exact IDs above also exclude hash collisions.- .whereRaw(- "xxHash32(e.trace_id) IN (SELECT arrayJoin(arrayMap(id -> xxHash32(id), {traceIds: Array(String)})))",- )- // One shared window enables partition/granule pruning. It can include- // observations beyond a selected trace's own bounds when another trace- // widens the batch window; per-trace predicates can narrow that coverage.+ .whereRaw(`(${hashPredicates.join(" OR ")})`)+ // Keep one shared outer window for partition/granule pruning..whereRaw("e.start_time >= fromUnixTimestamp64Milli({batchMinStart: Int64}) AND e.start_time <= fromUnixTimestamp64Milli({batchMaxStart: Int64})",{batchMinStart:- Math.min(...props.traces.map((trace) => trace.minStart)) -+ Math.min(...props.traces.map(({ minStart }) => minStart)) -TRACE_QUERY_BUFFER_MS,batchMaxStart:- Math.max(...props.traces.map((trace) => trace.maxStart)) ++ Math.max(...props.traces.map(({ maxStart }) => maxStart)) +TRACE_QUERY_BUFFER_MS,},+ )+ // Batch companions must not widen another trace's required coverage.+ // Identical project/time windows share one ID list; repeated pair intervals+ // remain separate OR branches and therefore return each matching row once.+ .whereRaw(+ `(${timeGroupPredicates.join(`+ OR `)})`,+ timeGroupParams,);- const { query, params } = builder.buildWithParams();+ return {+ ...builder.buildWithParams(),+ projectIds,+ };+};++/**+ * Stream full events_full payloads into the worker without retaining a whole batch.+ * Rows reflect the events table's current merge state, as in other event reads;+ * this read-only experiment does not deduplicate versions or enrich model data.+ */+export async function* getTraceBatchEventStream(+ props: TraceBatchEventStreamProps,+ options: {+ maxThreads?: number;+ maxBlockSize?: number;+ experimentId?: string;+ } = {},+): AsyncGenerator<TraceBatchEventRow> {+ if (props.traces.length === 0) return;++ const { query, params, projectIds } = buildTraceBatchEventQuery(props);yield* queryClickhouseStream<TraceBatchEventRow>({query,params,+ useMultipartParamsAuto: true,tags: {projectId: projectIds.length === 1 ? projectIds[0] : "MULTI_PROJECT",+ ...(options.experimentId ? { experimentId: options.experimentId } : {}),},preferredClickhouseService: "EventsReadOnly",+ clickhouseConfigs: {+ compression: { response: true },+ },// Bound background-read CPU/time; timeouts fail instead of returning partial results.clickhouseSettings: {- max_threads: 2,+ max_threads: options.maxThreads ?? 2,+ ...(options.maxBlockSize === undefined+ ? {}+ : { max_block_size: String(options.maxBlockSize) }),max_execution_time: 30,timeout_overflow_mode: "throw",},diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex 564c57d035..6dcb82efce 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -65,7 +65,7 @@ const findFilterOption = () => rows.find((row) => row.column === column && row.value === value);describe("Clickhouse Events Repository Test", () => {- it("streams complete trace batches with full payloads, exact tenant/trace pairs and shared batch time bounds", async () => {+ it("streams complete trace batches with full payloads and exact per-trace time bounds", async () => {const batchProjectId = randomUUID();const otherProjectId = randomUUID();const firstTraceId = randomUUID();@@ -93,26 +93,44 @@ describe("Clickhouse Events Repository Test", () => {trace_id: secondTraceId,start_time: new Date(start + 600_000 + buffer),});- // Another trace can widen the shared window beyond this trace's own bounds.- const insideBatchWindow = createEvent({+ const longTraceUpperEndpoint = createEvent({+ project_id: batchProjectId,+ trace_id: secondTraceId,+ start_time: new Date(start + 1_800_000 + buffer),+ });+ const companionOnlyRow = createEvent({project_id: batchProjectId,trace_id: firstTraceId,start_time: new Date(start + buffer + 1),});+ const repeatedIntervalRow = createEvent({+ project_id: batchProjectId,+ trace_id: firstTraceId,+ start_time: new Date(start + 1_200_000),+ });+ const otherProjectWindowRow = createEvent({+ project_id: otherProjectId,+ trace_id: firstTraceId,+ start_time: new Date(start),+ });await createEventsCh([first,second,- insideBatchWindow,+ longTraceUpperEndpoint,+ companionOnlyRow,+ repeatedIntervalRow,+ // This row only matches the same trace ID's window in another project.+ otherProjectWindowRow,createEvent({project_id: otherProjectId,trace_id: otherTraceId,start_time: new Date(start),}),- // The same trace ID in two projects represents two different traces.+ // The same trace ID in two projects has independent time windows.createEvent({project_id: otherProjectId,trace_id: firstTraceId,- start_time: new Date(start),+ start_time: new Date(start + 600_000),}),// These crossed pairs match both independent IN lists, but are unrequested.createEvent({@@ -143,7 +161,7 @@ describe("Clickhouse Events Repository Test", () => {createEvent({project_id: batchProjectId,trace_id: secondTraceId,- start_time: new Date(start + 600_000 + buffer + 1),+ start_time: new Date(start + 1_800_000 + buffer + 1),}),]);@@ -169,7 +187,9 @@ describe("Clickhouse Events Repository Test", () => {let rowCount = 0;let fullPayloadSeen = false;- let insideBatchWindowSeen = false;+ let companionOnlyRowSeen = false;+ let otherProjectWindowRowSeen = false;+ let repeatedIntervalRowCount = 0;const foundTraces = new Set<string>();for await (const event of getTraceBatchEventStream({traces: [@@ -183,7 +203,7 @@ describe("Clickhouse Events Repository Test", () => {projectId: batchProjectId,traceId: secondTraceId,minStart: start + 600_000,- maxStart: start + 600_000,+ maxStart: start + 1_800_000,},{projectId: otherProjectId,@@ -194,12 +214,25 @@ describe("Clickhouse Events Repository Test", () => {{projectId: otherProjectId,traceId: firstTraceId,+ minStart: start + 600_000,+ maxStart: start + 600_000,+ },+ // Repeated identical intervals must not duplicate output rows.+ {+ projectId: batchProjectId,+ traceId: firstTraceId,minStart: start,maxStart: start,},- // Exercise the maximum configurable batch size. Missing IDs stress- // HTTP parameters and query size without unrelated fixture rows.- ...Array.from({ length: 996 }, () => ({+ // A repeated pair may contribute a separate required interval.+ {+ projectId: batchProjectId,+ traceId: firstTraceId,+ minStart: start + 1_200_000,+ maxStart: start + 1_200_000,+ },+ // Missing IDs stress HTTP parameters without unrelated fixture rows.+ ...Array.from({ length: 994 }, () => ({projectId: batchProjectId,traceId: randomUUID(),minStart: start,@@ -209,8 +242,14 @@ describe("Clickhouse Events Repository Test", () => {})) {rowCount++;foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));- if (event.span_id === insideBatchWindow.span_id) {- insideBatchWindowSeen = true;+ if (event.span_id === companionOnlyRow.span_id) {+ companionOnlyRowSeen = true;+ }+ if (event.span_id === otherProjectWindowRow.span_id) {+ otherProjectWindowRowSeen = true;+ }+ if (event.span_id === repeatedIntervalRow.span_id) {+ repeatedIntervalRowCount++;}if (event.span_id === first.span_id) {fullPayloadSeen = true;@@ -225,7 +264,9 @@ describe("Clickhouse Events Repository Test", () => {}}expect(fullPayloadSeen).toBe(true);- expect(insideBatchWindowSeen).toBe(true);+ expect(companionOnlyRowSeen).toBe(false);+ expect(otherProjectWindowRowSeen).toBe(false);+ expect(repeatedIntervalRowCount).toBe(1);expect(foundTraces).toEqual(new Set([JSON.stringify([batchProjectId, firstTraceId]),@@ -234,9 +275,39 @@ describe("Clickhouse Events Repository Test", () => {JSON.stringify([otherProjectId, firstTraceId]),]),);- expect(rowCount).toBe(extraRowCount + 5);+ expect(rowCount).toBe(extraRowCount + 6);}, 60_000);+ it("returns matching rows across parameter chunks in a 10,000-trace batch", async () => {+ const start = Date.now();+ const traces = Array.from({ length: 10_000 }, (_, index) => ({+ projectId: randomUUID(),+ traceId: randomUUID(),+ minStart: start + index,+ maxStart: start + index,+ }));+ // Exercise both sides of time-group, pair, and ID chunk boundaries, plus+ // the final chunk. Distinct projects also exercise project parameter chunks.+ const expectedRows = [0, 49, 50, 999, 1_000, 1_999, 2_000, 9_999].map(+ (index) =>+ createEvent({+ project_id: traces[index].projectId,+ trace_id: traces[index].traceId,+ start_time: new Date(traces[index].minStart),+ }),+ );+ await createEventsCh(expectedRows);++ const foundSpanIds: string[] = [];+ for await (const event of getTraceBatchEventStream({ traces })) {+ foundSpanIds.push(event.span_id);+ }++ expect(foundSpanIds.sort()).toEqual(+ expectedRows.map(({ span_id }) => span_id).sort(),+ );+ }, 120_000);+it("should kill redis connection", () => {// we need at least one test case to avoid hanging// redis connection when everything else is skipped.diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex 5fdbfd51a7..c0310a725f 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -20,7 +20,8 @@`src/queues/traceBatchQueue.ts`; controls and Redis lifecycle are documented in`src/features/traceBatching/README.md`. Keep producer, dispatcher, consumer and readsindependently default-off and cloud-gated. Do not expose these PoC controls in- local or production env templates.+ local or production env templates. Reader query controls are independent of+ locality selection; logs must preserve separate input/output/metadata metrics.- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.- Service layer: `src/services/*`- Rust addon (`@langfuse/native`): telemetry init and the startup hello call livediff --git a/worker/src/env.ts b/worker/src/env.tsindex 5bd76cfa9a..482d20dd59 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -126,11 +126,27 @@ const EnvSchema = z.object({.int().positive().default(2),+ LANGFUSE_TRACE_BATCH_MAX_THREADS: z.coerce+ .number()+ .int()+ .positive()+ .default(2),+ LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: z.coerce+ .number()+ .int()+ .positive()+ .optional(),+ LANGFUSE_TRACE_BATCH_EXPERIMENT_ID: z+ .string()+ .min(1)+ .max(64)+ .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/)+ .optional(),LANGFUSE_TRACE_BATCH_MAX_SIZE: z.coerce.number().int().positive()- .max(1_000)+ .max(10_000).default(60),LANGFUSE_TRACE_BATCH_STRATEGY: z.enum(["project", "locality"])diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 5462ff206e..1f4f235732 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -7,19 +7,22 @@ changes. These internal controls are intentionally absent from env templates.## Controls-| Variable | Default | Role |-| --------------------------------------------- | --------- | ------------------------------------------------------------- |-| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |-| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |-| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |-| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |-| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |-| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |-| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 1,000. |-| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |-| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |-| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |+| Variable | Default | Role |+| --------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------- |+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |Producer, dispatcher and consumer can run independently. Sampling reuses theevaluator's deterministic trace-ID sampler; every observation of the same trace@@ -63,13 +66,45 @@ Sizes count UTF-8 bytes, including metadata keys and values, excluding JSONtransport framing/escaping and compression. They do not measure ClickHouse bytesscanned or per-trace ingestion size; retries can record another batch sample.Exact `(projectId, traceId)`-pairs protect tenant boundaries; project, trace hash and a shared batch time-window prune reads. `TRACE_QUERY_BUFFER_MS` pads the earliest and latest recorded-start times by two minutes each. This fixed query margin can include observations-outside the recorded bounds; it neither waits for arrivals nor guarantees trace-completeness. Query limits are fixed at two threads and 30 seconds with-timeouts failing the job. Per-trace windows, query tuning controls and allocation-based on trace size are separate experiments.+pairs protect tenant boundaries; project, trace hash and a shared outer time+window prune reads. Each selected trace also gets its own interval, so unrelated+batch companions cannot widen the payload returned for that trace.+`TRACE_QUERY_BUFFER_MS` pads each recorded interval by two minutes on both sides.+This fixed margin neither waits for arrivals nor guarantees trace completeness;+repeated intervals for the same pair form a union without duplicating rows.+Queries use response compression and chunked multipart parameters for up to+10,000 UUID-sized trace/project IDs. The fixed 30-second timeout fails the job+rather than accepting partial results. This does not cap observation count,+query memory or dispatcher snapshot memory.++## Compare reader settings++Establish a fresh control measurement after deploying this reader: both arms+must use the same per-trace windows and compression. Hold sampling, batch cap,+concurrency, threads, idle delay and traffic cohort constant; then change only+block size (and the identifying label):++| Variable | Control | Treatment |+| ------------------------------------- | --------------------- | ----------------- |+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | `2` |+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | `512` |+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | `trace-block-control` | `trace-block-512` |++Run these settings only in cloud workers with the existing four enablement+flags explicitly set to `true`. Changing a query setting does not enable any+part of the pipeline. `max_block_size` is a row-count hint, not a byte or memory+limit; large inputs may still dominate memory. Keep the initial cap at 120 when+comparing existing experiments; raising the supported ceiling is not a rollout+recommendation.++With an experiment ID, each read attempt logs its label, build ID, configured+threads/block size, batch size, concurrency, sampling and timing controls before+querying, including failed attempts. The same ID enters ClickHouse+`log_comment` alongside existing surface/route/project attribution. Leave the ID+unset to suppress these per-attempt logs. Strategy attribution uses the+dispatcher metrics described below. Compare+query memory/latency and worker memory as well as input/output/metadata byte+metrics. Redis storage load tests do not measure selector or query memory.### Opt-in locality grouping@@ -144,5 +179,4 @@ lazy retention rules. Drain and expired jobs are removed immediately on success.For an already enabled cloud experiment, explicitly set the new read flag to`true` when deploying this version if reads should continue; leaving it unsetdrains instead. Upgrade all producers before relying on native expiry: older-producers do not install or refresh that TTL. Query controls and load-test artifacts belong in separate follow-up changes.+producers do not install or refresh that TTL.diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex cdb29bed53..6edd7b65ec 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { type Job } from "bullmq";import {getTraceBatchEventStream,+ logger,QueueJobs,type QueueName,recordDistribution,@@ -74,6 +75,84 @@ describe("trace batch queue", () => {expect(job.opts.removeOnComplete).toBe(true);});+ it("forwards overrides and rejects a partially consumed stream without reporting success", async () => {+ const originalEnv = { ...env };+ Object.assign(env, {+ LANGFUSE_TRACE_BATCH_MAX_THREADS: 1,+ LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: 512,+ LANGFUSE_TRACE_BATCH_EXPERIMENT_ID: "arm-b",+ BUILD_ID: "test-build",+ });+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);+ const failure = new Error("query failed");+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {+ yield {+ project_id: "project",+ trace_id: "trace",+ span_id: "span",+ parent_span_id: null,+ start_time: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000000",+ type: "GENERATION",+ name: "generation",+ input: "input",+ output: "output",+ metadata: {},+ tool_definitions: {},+ tool_calls: [],+ tool_call_names: [],+ };+ throw failure;+ });+ const payload = {+ traces: [+ {+ projectId: "project",+ traceId: "trace",+ minStart: 1_000,+ maxStart: 2_000,+ revision: "r",+ },+ ],+ };+ const job = {+ data: {+ id: "batch",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(),+ payload,+ },+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ try {+ await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(+ failure,+ );+ expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload, {+ maxThreads: 1,+ maxBlockSize: 512,+ experimentId: "arm-b",+ });+ expect(log).toHaveBeenCalledWith(+ "Trace batch experiment read",+ expect.objectContaining({+ experimentId: "arm-b",+ buildId: "test-build",+ maxThreads: 1,+ maxBlockSize: 512,+ batchTraceCount: 1,+ }),+ );+ expect(recordDistribution).not.toHaveBeenCalled();+ } finally {+ env.LANGFUSE_TRACE_BATCH_MAX_THREADS =+ originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;+ env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE =+ originalEnv.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE;+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID =+ originalEnv.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;+ env.BUILD_ID = originalEnv.BUILD_ID;+ }+ });it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {const ingestionEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;const dispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;@@ -147,7 +226,10 @@ describe("trace batch queue", () => {);}expect(getTraceBatchEventStream).toHaveBeenCalledTimes(2);- expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload);+ expect(getTraceBatchEventStream).toHaveBeenCalledWith(+ payload,+ expect.anything(),+ );expect(yieldedRows).toBe(8);for (const [name, bytes] of [["input_bytes", 20],@@ -246,9 +328,10 @@ describe("trace batch queue", () => {"langfuse.trace_batch.found_project_count",Number(hasRows),);- expect(getTraceBatchEventStream).toHaveBeenCalledWith({- traces: [{ ...trace, projectId: "project" }],- });+ expect(getTraceBatchEventStream).toHaveBeenCalledWith(+ { traces: [{ ...trace, projectId: "project" }] },+ expect.anything(),+ );},);});diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex 522b1c4278..834f0b4353 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -1,6 +1,7 @@import { type Processor } from "bullmq";import {getTraceBatchEventStream,+ logger,recordDistribution,recordIncrement,TraceBatchEventSchema,@@ -43,7 +44,27 @@ export const traceBatchQueueProcessor: Processor<let outputBytes = 0;let metadataBytes = 0;- for await (const event of getTraceBatchEventStream(batch)) {+ const queryOptions = {+ maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,+ maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,+ experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,+ };+ if (queryOptions.experimentId) {+ logger.info("Trace batch experiment read", {+ ...queryOptions,+ // null means no override; the server profile's value is not known here.+ maxBlockSize: queryOptions.maxBlockSize ?? null,+ buildId: env.BUILD_ID,+ batchTraceCount: batch.traces.length,+ maxBatchSize: env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,+ samplingRate: env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE,+ idleMs: env.LANGFUSE_TRACE_BATCH_IDLE_MS,+ dispatchIntervalMs: env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS,+ });+ }++ for await (const event of getTraceBatchEventStream(batch, queryOptions)) {observationCount++;foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));foundProjects.add(event.project_id);