Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): Add cloud-only trace batch PoC (baseline) (#17495)

Lysander Kiesel0e62b3f68bf5
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index 55a6c4ccc5..37bd3f3d90 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -23,6 +23,10 @@
- Repository layer: `src/server/repositories/*`
- Queue payload schemas: `src/server/queues.ts`
- Queue helpers: `src/server/redis/*`
+- Internal trace-batch queue: `src/server/redis/traceBatch.ts` (cloud-gated);
+ payloads in `src/server/queues.ts` accept persisted single-project jobs.
+ Full-event streaming reads live in `src/server/repositories/trace-batch.ts`;
+ the worker owns experiment enablement and lifecycle.
- Code evaluator dispatcher/error contract: `src/server/evals/codeEvalDispatcherTypes.ts`. Keep provider mappings, user-visible messages, and worker terminal-outcome classification aligned when adding an error code.
- Dashboard/monitor query feature (data model + server-only builder/executor): `src/features/query/*`
- Query-builder AST (server half, WIP): `src/server/query-ast/*` — golden-SQL
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index ce1e6a7796..adc85bfb14 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -101,6 +101,7 @@ export * from "./redis/webhookQueue";
export * from "./redis/monitorQueue";
export * from "./redis/inAppAgentRunQueue";
export * from "./redis/traceDelete";
+export * from "./redis/traceBatch";
export * from "./redis/projectDelete";
export * from "./redis/scoreDelete";
export * from "./redis/datasetDelete";
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index eadebcf135..afaaccb62d 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -595,7 +595,7 @@ const AGGREGATION_FIELD_SETS = {
* // Use when you need to query across all projects (use with caution!)
* const builder = new EventsQueryBuilder({ projectId: NoProjectId });
*/
-const NoProjectId = Symbol("NoProjectId");
+export const NoProjectId = Symbol("NoProjectId");
export type NoProjectIdType = typeof NoProjectId;
/**
diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.ts
index 7d995ea025..531964244f 100644
--- a/packages/shared/src/server/queues.ts
+++ b/packages/shared/src/server/queues.ts
@@ -404,6 +404,7 @@ export const RetryBaggage = z.object({
export type RetryBaggage = z.infer<typeof RetryBaggage>;
export enum QueueName {
+ TraceBatch = "trace-batch",
TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
TraceDelete = "trace-delete",
ProjectDelete = "project-delete",
@@ -446,6 +447,7 @@ export enum QueueName {
}
export enum QueueJobs {
+ TraceBatch = "trace-batch",
TraceUpsert = "trace-upsert",
TraceDelete = "trace-delete",
ProjectDelete = "project-delete",
@@ -484,7 +486,37 @@ export enum QueueJobs {
V4LegacyApiUsageJob = "v4-legacy-api-usage-job",
}
+export const TraceBatchTraceSchema = z.object({
+ projectId: z.string(),
+ traceId: z.string(),
+ minStart: z.number(),
+ maxStart: z.number(),
+ revision: z.string(),
+});
+
+export const TraceBatchEventSchema = z.object({
+ timestamp: z.coerce.date(),
+ id: z.string(),
+ name: z.literal(QueueJobs.TraceBatch),
+ payload: z.union([
+ z.object({ traces: z.array(TraceBatchTraceSchema).min(1) }).strict(),
+ // Persisted single-project jobs must remain readable while consumers drain.
+ z
+ .object({
+ projectId: z.string(),
+ traces: z
+ .array(TraceBatchTraceSchema.omit({ projectId: true }).strict())
+ .min(1),
+ })
+ .strict()
+ .transform(({ projectId, traces }) => ({
+ traces: traces.map((trace) => ({ ...trace, projectId })),
+ })),
+ ]),
+});
+
export type TQueueJobTypes = {
+ [QueueName.TraceBatch]: z.infer<typeof TraceBatchEventSchema>;
[QueueName.TraceUpsert]: {
timestamp: Date;
id: string;
diff --git a/packages/shared/src/server/redis/getQueue.ts b/packages/shared/src/server/redis/getQueue.ts
index 4e65c0bfa0..62dc54f851 100644
--- a/packages/shared/src/server/redis/getQueue.ts
+++ b/packages/shared/src/server/redis/getQueue.ts
@@ -7,6 +7,7 @@ import { CloudFreeTierUsageThresholdQueue } from "./cloudFreeTierUsageThresholdQ
import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
import { ExperimentCreateQueue } from "./experimentCreateQueue";
import { TraceDeleteQueue } from "./traceDelete";
+import { TraceBatchQueue } from "./traceBatch";
import { ProjectDeleteQueue } from "./projectDelete";
import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
import { PostHogIntegrationProcessingQueue } from "./postHogIntegrationProcessingQueue";
@@ -48,6 +49,8 @@ export function getQueue(
>,
): Queue | null {
switch (queueName) {
+ case QueueName.TraceBatch:
+ return TraceBatchQueue.getInstance();
case QueueName.BatchExport:
return BatchExportQueue.getInstance();
case QueueName.CloudUsageMeteringQueue:
diff --git a/packages/shared/src/server/redis/traceBatch.test.ts b/packages/shared/src/server/redis/traceBatch.test.ts
new file mode 100644
index 0000000000..77553739fa
--- /dev/null
+++ b/packages/shared/src/server/redis/traceBatch.test.ts
@@ -0,0 +1,45 @@
+import { expect, it, vi } from "vitest";
+
+const { env, createQueueOptions, queueCreated } = vi.hoisted(() => ({
+ env: {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined as string | undefined,
+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: "true",
+ LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED: "true",
+ QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED: "true",
+ LANGFUSE_TRACE_BATCH_READ_ENABLED: "true",
+ },
+ createQueueOptions: vi.fn(() => ({ connection: {} })),
+ queueCreated: vi.fn(),
+}));
+
+vi.mock("../../env", () => ({ env }));
+vi.mock("./redis", () => ({
+ createBullMQQueueOptionsWithRedis: createQueueOptions,
+}));
+vi.mock("../logger", () => ({ logger: { error: vi.fn() } }));
+vi.mock("bullmq", () => ({
+ Queue: class {
+ constructor() {
+ queueCreated();
+ }
+ on = vi.fn();
+ },
+}));
+
+import { TraceBatchQueue } from "./traceBatch";
+
+it("keeps queue inspection inert outside cloud even with all experiment flags enabled", () => {
+ expect(TraceBatchQueue.getInstance()).toBeNull();
+ expect(createQueueOptions).not.toHaveBeenCalled();
+ expect(queueCreated).not.toHaveBeenCalled();
+
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+ const cloudQueue = TraceBatchQueue.getInstance();
+ expect(cloudQueue).not.toBeNull();
+ expect(TraceBatchQueue.getInstance()).toBe(cloudQueue);
+ expect(queueCreated).toHaveBeenCalledTimes(1);
+
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+ expect(TraceBatchQueue.getInstance()).toBeNull();
+ expect(queueCreated).toHaveBeenCalledTimes(1);
+});
diff --git a/packages/shared/src/server/redis/traceBatch.ts b/packages/shared/src/server/redis/traceBatch.ts
new file mode 100644
index 0000000000..3c994ab9b4
--- /dev/null
+++ b/packages/shared/src/server/redis/traceBatch.ts
@@ -0,0 +1,35 @@
+import { Queue } from "bullmq";
+import { env } from "../../env";
+import { QueueName, type TQueueJobTypes } from "../queues";
+import { createBullMQQueueOptionsWithRedis } from "./redis";
+import { logger } from "../logger";
+
+export class TraceBatchQueue {
+ private static instance: Queue<TQueueJobTypes[QueueName.TraceBatch]> | null =
+ null;
+
+ public static getInstance() {
+ if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) return null;
+ if (this.instance) return this.instance;
+ const options = createBullMQQueueOptionsWithRedis(QueueName.TraceBatch);
+ if (!options) return null;
+
+ this.instance = new Queue<TQueueJobTypes[QueueName.TraceBatch]>(
+ QueueName.TraceBatch,
+ {
+ ...options,
+ defaultJobOptions: {
+ // Retain recent job IDs so a dispatcher retry can reuse its delivery.
+ removeOnComplete: { age: 3600, count: 10000 },
+ removeOnFail: 1000,
+ attempts: 3,
+ backoff: { type: "exponential", delay: 5000 },
+ },
+ },
+ );
+ this.instance.on("error", (error) => {
+ logger.error("TraceBatchQueue error", error);
+ });
+ return this.instance;
+ }
+}
diff --git a/packages/shared/src/server/repositories/index.ts b/packages/shared/src/server/repositories/index.ts
index c4e2d6924e..f55c331ee7 100644
--- a/packages/shared/src/server/repositories/index.ts
+++ b/packages/shared/src/server/repositories/index.ts
@@ -3,6 +3,7 @@ export * from "./traces";
export * from "./observations";
export * from "./events";
export * from "./events-stream";
+export * from "./trace-batch";
export * from "./types";
export * from "./dashboards";
export * from "./traces_converters";
diff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.ts
new file mode 100644
index 0000000000..74fbf53e74
--- /dev/null
+++ b/packages/shared/src/server/repositories/trace-batch.ts
@@ -0,0 +1,108 @@
+import {
+ EventsQueryBuilder,
+ NoProjectId,
+} from "../queries/clickhouse-sql/event-query-builder";
+import { queryClickhouseStream, TupleParam } from "./clickhouse";
+
+type TraceBatchEventRow = {
+ project_id: string;
+ trace_id: string;
+ span_id: string;
+ parent_span_id: string | null;
+ start_time: string;
+ event_ts: string;
+ type: string;
+ name: string;
+ input: string;
+ output: string;
+ metadata: Record<string, string>;
+ tool_definitions: Record<string, string>;
+ tool_calls: string[];
+ tool_call_names: string[];
+};
+
+const TRACE_QUERY_BUFFER_MS = 2 * 60_000;
+
+/**
+ * Stream full events_full payloads into the worker without retaining a whole batch.
+ * Rows reflect the events table's current merge state, as in other event reads;
+ * this read-only experiment does not deduplicate versions or enrich model data.
+ */
+export async function* getTraceBatchEventStream(props: {
+ traces: ReadonlyArray<{
+ projectId: string;
+ traceId: string;
+ minStart: number;
+ maxStart: number;
+ }>;
+}): AsyncGenerator<TraceBatchEventRow> {
+ if (props.traces.length === 0) return;
+
+ const projectIds = [...new Set(props.traces.map((trace) => trace.projectId))];
+ const traceIds = [...new Set(props.traces.map((trace) => trace.traceId))];
+ const builder = new EventsQueryBuilder({ projectId: NoProjectId })
+ .selectRaw(
+ "e.project_id",
+ "e.trace_id",
+ "e.span_id",
+ "e.parent_span_id",
+ "e.start_time",
+ "e.event_ts",
+ "e.type",
+ "e.name",
+ )
+ // Load full input/output (false = no truncation) and every metadata key.
+ .selectIO(false)
+ .selectMetadataExpanded()
+ .selectFieldSet("tools")
+ // Read events_full rather than the smaller, truncated events_core table.
+ .forceFullTable()
+ // Separate filters retain project-prefix and trace-index pruning.
+ .whereRaw("e.project_id IN ({projectIds: Array(String)})", { projectIds })
+ .whereRaw("e.trace_id IN ({traceIds: Array(String)})", { traceIds })
+ // Independent IN lists also match crossed pairs. Only these exact tenant/
+ // trace pairs may return payloads, even when projects share trace IDs.
+ .whereRaw(
+ "(e.project_id, e.trace_id) IN {tracePairs: Array(Tuple(String, String))}",
+ {
+ tracePairs: props.traces.map(
+ ({ projectId, traceId }) => new TupleParam([projectId, traceId]),
+ ),
+ },
+ )
+ // Equality filters add this primary-key hash condition automatically; IN
+ // needs it explicitly. The exact IDs above also exclude hash collisions.
+ .whereRaw(
+ "xxHash32(e.trace_id) IN (SELECT arrayJoin(arrayMap(id -> xxHash32(id), {traceIds: Array(String)})))",
+ )
+ // One shared window enables partition/granule pruning. It can include
+ // observations beyond a selected trace's own bounds when another trace
+ // widens the batch window; per-trace predicates can narrow that coverage.
+ .whereRaw(
+ "e.start_time >= fromUnixTimestamp64Milli({batchMinStart: Int64}) AND e.start_time <= fromUnixTimestamp64Milli({batchMaxStart: Int64})",
+ {
+ batchMinStart:
+ Math.min(...props.traces.map((trace) => trace.minStart)) -
+ TRACE_QUERY_BUFFER_MS,
+ batchMaxStart:
+ Math.max(...props.traces.map((trace) => trace.maxStart)) +
+ TRACE_QUERY_BUFFER_MS,
+ },
+ );
+
+ const { query, params } = builder.buildWithParams();
+ yield* queryClickhouseStream<TraceBatchEventRow>({
+ query,
+ params,
+ tags: {
+ projectId: projectIds.length === 1 ? projectIds[0] : "MULTI_PROJECT",
+ },
+ preferredClickhouseService: "EventsReadOnly",
+ // Bound background-read CPU/time; timeouts fail instead of returning partial results.
+ clickhouseSettings: {
+ max_threads: 2,
+ max_execution_time: 30,
+ timeout_overflow_mode: "throw",
+ },
+ });
+}
diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.ts
index 79a1dc0775..564c57d035 100644
--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts
@@ -1,6 +1,7 @@
import {
createEvent,
createEventsCh,
+ getTraceBatchEventStream,
getObservationsForTraceFromEventsTable,
getObservationsWithModelDataFromEventsTable,
getObservationsCountFromEventsTable,
@@ -64,6 +65,178 @@ const findFilterOption = (
) => rows.find((row) => row.column === column && row.value === value);
describe("Clickhouse Events Repository Test", () => {
+ it("streams complete trace batches with full payloads, exact tenant/trace pairs and shared batch time bounds", async () => {
+ const batchProjectId = randomUUID();
+ const otherProjectId = randomUUID();
+ const firstTraceId = randomUUID();
+ const secondTraceId = randomUUID();
+ const otherTraceId = randomUUID();
+ const start = Date.now();
+ const buffer = 2 * 60_000;
+ const input = JSON.stringify({ message: "input".repeat(100) });
+ const output = JSON.stringify({ message: "output".repeat(100) });
+ const metadataValue = JSON.stringify({ detail: "metadata".repeat(100) });
+ const first = createEvent({
+ project_id: batchProjectId,
+ trace_id: firstTraceId,
+ start_time: new Date(start - buffer),
+ input,
+ output,
+ metadata_names: ["context"],
+ metadata_values: [metadataValue],
+ tool_definitions: { search: '{"description":"Search"}' },
+ tool_calls: ['{"name":"search"}'],
+ tool_call_names: ["search"],
+ });
+ const second = createEvent({
+ project_id: batchProjectId,
+ trace_id: secondTraceId,
+ start_time: new Date(start + 600_000 + buffer),
+ });
+ // Another trace can widen the shared window beyond this trace's own bounds.
+ const insideBatchWindow = createEvent({
+ project_id: batchProjectId,
+ trace_id: firstTraceId,
+ start_time: new Date(start + buffer + 1),
+ });
+ await createEventsCh([
+ first,
+ second,
+ insideBatchWindow,
+ createEvent({
+ project_id: otherProjectId,
+ trace_id: otherTraceId,
+ start_time: new Date(start),
+ }),
+ // The same trace ID in two projects represents two different traces.
+ createEvent({
+ project_id: otherProjectId,
+ trace_id: firstTraceId,
+ start_time: new Date(start),
+ }),
+ // These crossed pairs match both independent IN lists, but are unrequested.
+ createEvent({
+ project_id: batchProjectId,
+ trace_id: otherTraceId,
+ start_time: new Date(start),
+ }),
+ createEvent({
+ project_id: otherProjectId,
+ trace_id: secondTraceId,
+ start_time: new Date(start),
+ }),
+ createEvent({
+ project_id: randomUUID(),
+ trace_id: firstTraceId,
+ start_time: new Date(start),
+ }),
+ createEvent({
+ project_id: batchProjectId,
+ trace_id: randomUUID(),
+ start_time: new Date(start),
+ }),
+ createEvent({
+ project_id: batchProjectId,
+ trace_id: firstTraceId,
+ start_time: new Date(start - buffer - 1),
+ }),
+ createEvent({
+ project_id: batchProjectId,
+ trace_id: secondTraceId,
+ start_time: new Date(start + 600_000 + buffer + 1),
+ }),
+ ]);
+
+ // Exceed the single-trace reader's historical 20,000-observation cap.
+ const extraRowCount = 20_001;
+ const template = createEvent({
+ project_id: batchProjectId,
+ trace_id: firstTraceId,
+ start_time: new Date(start),
+ input: "",
+ output: "",
+ metadata_names: [],
+ metadata_values: [],
+ });
+ for (let offset = 0; offset < extraRowCount; offset += 2_000) {
+ await createEventsCh(
+ Array.from({ length: Math.min(2_000, extraRowCount - offset) }, () => {
+ const spanId = randomUUID();
+ return { ...template, span_id: spanId, id: spanId };
+ }),
+ );
+ }
+
+ let rowCount = 0;
+ let fullPayloadSeen = false;
+ let insideBatchWindowSeen = false;
+ const foundTraces = new Set<string>();
+ for await (const event of getTraceBatchEventStream({
+ traces: [
+ {
+ projectId: batchProjectId,
+ traceId: firstTraceId,
+ minStart: start,
+ maxStart: start,
+ },
+ {
+ projectId: batchProjectId,
+ traceId: secondTraceId,
+ minStart: start + 600_000,
+ maxStart: start + 600_000,
+ },
+ {
+ projectId: otherProjectId,
+ traceId: otherTraceId,
+ minStart: start,
+ maxStart: start,
+ },
+ {
+ projectId: otherProjectId,
+ traceId: firstTraceId,
+ minStart: start,
+ maxStart: start,
+ },
+ // Exercise the maximum configurable batch size. Missing IDs stress
+ // HTTP parameters and query size without unrelated fixture rows.
+ ...Array.from({ length: 996 }, () => ({
+ projectId: batchProjectId,
+ traceId: randomUUID(),
+ minStart: start,
+ maxStart: start,
+ })),
+ ],
+ })) {
+ rowCount++;
+ foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));
+ if (event.span_id === insideBatchWindow.span_id) {
+ insideBatchWindowSeen = true;
+ }
+ if (event.span_id === first.span_id) {
+ fullPayloadSeen = true;
+ expect(event).toMatchObject({
+ input,
+ output,
+ metadata: { context: metadataValue },
+ tool_definitions: first.tool_definitions,
+ tool_calls: first.tool_calls,
+ tool_call_names: first.tool_call_names,
+ });
+ }
+ }
+ expect(fullPayloadSeen).toBe(true);
+ expect(insideBatchWindowSeen).toBe(true);
+ expect(foundTraces).toEqual(
+ new Set([
+ JSON.stringify([batchProjectId, firstTraceId]),
+ JSON.stringify([batchProjectId, secondTraceId]),
+ JSON.stringify([otherProjectId, otherTraceId]),
+ JSON.stringify([otherProjectId, firstTraceId]),
+ ]),
+ );
+ expect(rowCount).toBe(extraRowCount + 5);
+ }, 60_000);
+
it("should kill redis connection", () => {
// we need at least one test case to avoid hanging
// redis connection when everything else is skipped.
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index cba0983dd7..5fdbfd51a7 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -16,6 +16,11 @@
- Worker registration/lifecycle: `src/queues/workerManager.ts`
- Queue processors: `src/queues/*`
- Feature processors: `src/features/*`
+- Internal cloud trace batching: `src/features/traceBatching/traceBatching.ts` and
+ `src/queues/traceBatchQueue.ts`; controls and Redis lifecycle are documented in
+ `src/features/traceBatching/README.md`. Keep producer, dispatcher, consumer and reads
+ independently default-off and cloud-gated. Do not expose these PoC controls in
+ local or production env templates.
- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.
- Service layer: `src/services/*`
- Rust addon (`@langfuse/native`): telemetry init and the startup hello call live
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
new file mode 100644
index 0000000000..e52ccf7f4c
--- /dev/null
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -0,0 +1,954 @@
+import { randomBytes, randomUUID } from "node:crypto";
+import { Queue, QueueEvents, Worker } from "bullmq";
+import {
+ afterAll,
+ afterEach,
+ beforeEach,
+ describe,
+ expect,
+ it,
+ vi,
+} from "vitest";
+import {
+ createNewRedisInstance,
+ getQueuePrefix,
+ getS3EventStorageClient,
+ getTraceBatchEventStream,
+ QueueJobs,
+ QueueName,
+ recordDistribution,
+ recordIncrement,
+ redis,
+ TraceBatchEventSchema,
+ TraceBatchQueue,
+ type TQueueJobTypes,
+} from "@langfuse/shared/src/server";
+import * as shared from "@langfuse/shared/src/server";
+import { env } from "../env";
+import {
+ trackTraceBatchActivity,
+ TraceBatchDispatcher,
+} from "../features/traceBatching/traceBatching";
+import { otelIngestionQueueProcessorBuilder } from "../queues/otelIngestionQueue";
+import { traceBatchQueueProcessor } from "../queues/traceBatchQueue";
+import { ClickhouseWriter } from "../services/ClickhouseWriter";
+
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
+ const original =
+ await importOriginal<typeof import("@langfuse/shared/src/server")>();
+ return {
+ ...original,
+ redis: original.createNewRedisInstance({
+ keyPrefix: `trace-batch-test-${randomUUID()}:`,
+ }),
+ recordDistribution: vi.fn(),
+ recordIncrement: vi.fn(),
+ getS3EventStorageClient: vi.fn(),
+ };
+});
+
+vi.mock("../features/evaluation/observationEval", async (importOriginal) => ({
+ ...(await importOriginal<
+ typeof import("../features/evaluation/observationEval")
+ >()),
+ fetchObservationEvalRules: vi.fn().mockResolvedValue([]),
+}));
+
+describe("trace micro-batch scheduling with Redis", () => {
+ const dueKey = "{trace-batch}:due";
+ const stateKey = "{trace-batch}:state";
+ const originalEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;
+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+ const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
+ const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
+ const originalSamplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;
+ const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;
+ const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;
+ const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;
+ let queue: Queue<TQueueJobTypes[QueueName.TraceBatch]>;
+ let connection: NonNullable<ReturnType<typeof createNewRedisInstance>>;
+ const runners: TraceBatchDispatcher[] = [];
+ const client = () => {
+ if (!redis) throw new Error("Redis is required for this test");
+ return redis;
+ };
+ const member = (projectId: string, traceId: string) =>
+ JSON.stringify([projectId, traceId]);
+ const event = (traceId: string, start = 1_000_000) => ({
+ traceId,
+ startTimeISO: new Date(start).toISOString(),
+ });
+ const runner = () => {
+ const instance = new TraceBatchDispatcher();
+ runners.push(instance);
+ return instance;
+ };
+ async function makeDue(projectId: string, ...traceIds: string[]) {
+ for (const traceId of traceIds)
+ await client().zadd(
+ dueKey,
+ Date.now() - 10_000,
+ member(projectId, traceId),
+ );
+ }
+
+ beforeEach(async () => {
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "true";
+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;
+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 60;
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 7_200_000;
+ await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");
+ const redisConnection = createNewRedisInstance();
+ if (!redisConnection) throw new Error("Redis is required for this test");
+ connection = redisConnection;
+ const name = `trace-batch-test-${randomUUID()}`;
+ queue = new Queue(name, { connection, prefix: getQueuePrefix(name) });
+ await queue.waitUntilReady();
+ vi.spyOn(TraceBatchQueue, "getInstance").mockReturnValue(queue);
+ });
+
+ afterEach(async () => {
+ for (const dispatcher of runners) await dispatcher.drain();
+ runners.length = 0;
+ vi.restoreAllMocks();
+ vi.clearAllMocks();
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = originalEnabled;
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;
+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = originalSamplingRate;
+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;
+ await queue.obliterate({ force: true });
+ await queue.close();
+ connection.disconnect();
+ await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");
+ });
+ afterAll(() => client().disconnect());
+
+ it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+ const evaluate = vi.spyOn(client(), "eval");
+ const lock = vi.spyOn(client(), "set");
+ await trackTraceBatchActivity("project", [event("trace")]);
+ await runner()["execute"]();
+ expect(evaluate).not.toHaveBeenCalled();
+ expect(lock).not.toHaveBeenCalled();
+ expect(await client().exists(dueKey, stateKey)).toBe(0);
+ expect(await queue.count()).toBe(0);
+ });
+
+ it("ingests through BullMQ, postpones readiness on new activity, and consumes full ClickHouse payloads once due", async () => {
+ const original = {
+ LANGFUSE_TRACE_BATCH_IDLE_MS: env.LANGFUSE_TRACE_BATCH_IDLE_MS,
+ LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,
+ LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED:
+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,
+ LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED:
+ env.LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED,
+ };
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 4_000;
+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0.1;
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED = "false";
+ env.LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED = "false";
+
+ const ingestionName = `trace-batch-ingestion-test-${randomUUID()}`;
+ const ingestionQueue = new Queue<
+ TQueueJobTypes[QueueName.OtelIngestionQueue]
+ >(ingestionName, { connection, prefix: getQueuePrefix(ingestionName) });
+ const ingestionEvents = new QueueEvents(ingestionName, {
+ connection,
+ prefix: ingestionQueue.opts.prefix,
+ });
+ const ingestionWorker = new Worker(
+ ingestionName,
+ otelIngestionQueueProcessorBuilder(false),
+ { connection, prefix: ingestionQueue.opts.prefix },
+ );
+ const batchEvents = new QueueEvents(queue.name, {
+ connection,
+ prefix: queue.opts.prefix,
+ });
+ const batchWorker = new Worker(queue.name, traceBatchQueueProcessor, {
+ connection,
+ prefix: queue.opts.prefix,
+ });
+ const writer = ClickhouseWriter.getInstance();
+
+ try {
+ await Promise.all([
+ ingestionWorker.waitUntilReady(),
+ ingestionEvents.waitUntilReady(),
+ batchWorker.waitUntilReady(),
+ batchEvents.waitUntilReady(),
+ ]);
+ const projectId = randomUUID();
+ // The evaluator sampler assigns these scores 0.0593, 0.0291 and 0.7116.
+ const traceId = "00000000000000000000000000000001";
+ const otherTraceId = "00000000000000000000000000000003";
+ const excludedTraceId = "00000000000000000000000000000002";
+ const input = JSON.stringify({ message: "input".repeat(1_000) });
+ const output = JSON.stringify({ message: "output".repeat(1_000) });
+ const metadata = JSON.stringify({ context: "metadata".repeat(1_000) });
+ const nano = BigInt(Date.now()) * 1_000_000n;
+ const timestamp = {
+ low: Number(nano & 0xffffffffn),
+ high: Number(nano >> 32n),
+ unsigned: true,
+ };
+ async function ingest(traceIds: string[]) {
+ // S3 transport and unrelated evaluator configuration are stubbed;
+ // OTLP conversion, writer, Redis scripts and both queue workers are real.
+ vi.mocked(getS3EventStorageClient).mockReturnValue({
+ download: vi.fn().mockResolvedValue(
+ JSON.stringify([
+ {
+ resource: { attributes: [] },
+ scopeSpans: [
+ {
+ scope: { name: "langfuse-sdk", version: "4.0.0" },
+ spans: traceIds.map((id) => ({
+ traceId: Buffer.from(id, "hex").toJSON(),
+ spanId: randomBytes(8).toJSON(),
+ name: "trace-batch-integration",
+ kind: 1,
+ startTimeUnixNano: timestamp,
+ endTimeUnixNano: timestamp,
+ attributes: Object.entries({
+ "langfuse.observation.type": "span",
+ "langfuse.observation.input": input,
+ "langfuse.observation.output": output,
+ "langfuse.observation.metadata": metadata,
+ }).map(([key, value]) => ({
+ key,
+ value: { stringValue: value },
+ })),
+ })),
+ },
+ ],
+ },
+ ]),
+ ),
+ } as unknown as ReturnType<typeof getS3EventStorageClient>);
+ const job = await ingestionQueue.add(QueueJobs.OtelIngestionJob, {
+ id: randomUUID(),
+ timestamp: new Date(),
+ name: QueueJobs.OtelIngestionJob,
+ payload: {
+ data: { fileKey: "trace-batch-integration.json" },
+ authCheck: {
+ validKey: true,
+ scope: { projectId, orgId: randomUUID(), accessLevel: "project" },
+ },
+ ingestionVersion: "4",
+ },
+ });
+ await job.waitUntilFinished(ingestionEvents, 10_000);
+ }
+ async function waitUntilRedisTime(deadline: number) {
+ await expect
+ .poll(
+ async () => {
+ const [seconds, microseconds] = await client().time();
+ return (
+ Number(seconds) * 1_000 +
+ Math.floor(Number(microseconds) / 1_000)
+ );
+ },
+ { interval: 25, timeout: 10_000 },
+ )
+ .toBeGreaterThanOrEqual(deadline);
+ }
+
+ await ingest([traceId, excludedTraceId]);
+ expect(await client().hlen(stateKey)).toBe(1);
+ const firstDue = Number(
+ await client().zscore(dueKey, member(projectId, traceId)),
+ );
+ await waitUntilRedisTime(firstDue - 2_000);
+ // Leave ample time to check the old deadline even on a busy CI worker.
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;
+ await ingest([traceId, otherTraceId, excludedTraceId]);
+ const renewedDue = Number(
+ await client().zscore(dueKey, member(projectId, traceId)),
+ );
+ expect(renewedDue).toBeGreaterThan(firstDue);
+ expect(await client().hlen(stateKey)).toBe(2);
+
+ const dispatcher = runner();
+ await waitUntilRedisTime(firstDue);
+ await dispatcher.processBatch();
+ expect(await queue.getJobCounts("wait", "active", "completed")).toEqual({
+ wait: 0,
+ active: 0,
+ completed: 0,
+ });
+
+ await writer.flushAll(true);
+ // Sampling only gates readiness; excluded observations are still ingested.
+ let excludedObservations = 0;
+ for await (const observation of getTraceBatchEventStream({
+ traces: [
+ {
+ projectId,
+ traceId: excludedTraceId,
+ minStart: Number(nano / 1_000_000n),
+ maxStart: Number(nano / 1_000_000n),
+ },
+ ],
+ })) {
+ expect(observation.trace_id).toBe(excludedTraceId);
+ excludedObservations++;
+ }
+ expect(excludedObservations).toBe(2);
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0;
+ await waitUntilRedisTime(renewedDue);
+ await dispatcher.processBatch();
+ const [batch] = await queue.getJobs(["wait", "active", "completed"]);
+ expect(
+ batch.data.payload.traces.map((trace) => trace.traceId).sort(),
+ ).toEqual([traceId, otherTraceId].sort());
+ const result = await batch.waitUntilFinished(batchEvents, 10_000);
+ expect(result).toMatchObject({
+ observationCount: 3,
+ traceCount: 2,
+ projectCount: 1,
+ });
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.found_project_count",
+ 1,
+ );
+ // All three observations must carry their untruncated I/O and metadata.
+ expect(result.ioMetadataBytes).toBeGreaterThanOrEqual(
+ 3 * (Buffer.byteLength(input) + Buffer.byteLength(output) + 8_000),
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.size",
+ 2,
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.io_metadata_bytes",
+ result.ioMetadataBytes,
+ );
+ expect(await client().zcard(dueKey)).toBe(0);
+ expect(await client().hlen(stateKey)).toBe(0);
+ await dispatcher.processBatch();
+ expect(await queue.getCompletedCount()).toBe(1);
+ } finally {
+ try {
+ await Promise.all([ingestionWorker.close(), batchWorker.close()]);
+ await Promise.all([ingestionEvents.close(), batchEvents.close()]);
+ await ingestionQueue.obliterate({ force: true });
+ await ingestionQueue.close();
+ await writer.shutdown();
+ } finally {
+ Object.assign(env, original);
+ }
+ }
+ }, 30_000);
+
+ it("keeps stable nested trace samples across replay and counts decisions once per trace in each ingestion batch", async () => {
+ const traceIds = Array.from({ length: 1_000 }, (_, i) =>
+ i.toString(16).padStart(32, "0"),
+ );
+ let previous: string[] = [];
+ // Fixed fixtures pin the evaluator's sampling cohort, including both endpoints.
+ for (const [rate, expectedCount] of [
+ [0, 0],
+ [0.1, 78],
+ [0.1, 78],
+ [0.5, 478],
+ [1, 1_000],
+ ]) {
+ await client().del(dueKey, stateKey);
+ vi.mocked(recordIncrement).mockClear();
+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = rate;
+ traceIds.reverse();
+ await trackTraceBatchActivity(
+ "project",
+ traceIds.flatMap((id) => [
+ event(id, 100),
+ event(id, 200),
+ event(id, 150),
+ ]),
+ );
+ const selected = (await client().hkeys(stateKey)).sort();
+ expect(selected).toHaveLength(expectedCount);
+ expect(await client().zcard(dueKey)).toBe(expectedCount);
+ expect(selected).toEqual(expect.arrayContaining(previous));
+ if (previous.length === expectedCount) expect(selected).toEqual(previous);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.sampling_decisions",
+ expectedCount,
+ { decision: "selected" },
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.sampling_decisions",
+ 1_000 - expectedCount,
+ { decision: "excluded" },
+ );
+ previous = selected;
+ }
+ });
+
+ it("preserves min/max across concurrent out-of-order arrivals, resets readiness, and separates tenants", async () => {
+ await Promise.all(
+ [30, 0, 20, 10].map((start) =>
+ trackTraceBatchActivity("project", [event("trace", start)]),
+ ),
+ );
+ const state = JSON.parse(
+ (await client().hget(stateKey, member("project", "trace")))!,
+ );
+ expect(state).toMatchObject({ minStart: 0, maxStart: 30 });
+ await makeDue("project", "trace");
+ await trackTraceBatchActivity("project", [event("trace", 15)]);
+ expect(
+ Number(await client().zscore(dueKey, member("project", "trace"))),
+ ).toBeGreaterThan(Date.now() + 590_000);
+ await trackTraceBatchActivity("other", [event("trace", 100)]);
+ expect(
+ JSON.parse((await client().hget(stateKey, member("other", "trace")))!),
+ ).toMatchObject({ minStart: 100, maxStart: 100 });
+ await runner().processBatch();
+ expect(await queue.getWaitingCount()).toBe(0);
+ });
+
+ it.each([
+ { maxSize: 60, sizes: [22, 60, 60, 60], projectCounts: [1, 1, 1, 2] },
+ { maxSize: 80, sizes: [42, 80, 80], projectCounts: [1, 1, 2] },
+ ])(
+ "dispatches cross-project batches capped at $maxSize, records their distribution, and drains with intake off",
+ async ({ maxSize, sizes, projectCounts }) => {
+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = maxSize;
+ const traces = Array.from(
+ { length: 201 },
+ (_, index) => `trace-${index}`,
+ );
+ await trackTraceBatchActivity(
+ "project",
+ traces.map((traceId) => event(traceId)),
+ );
+ await trackTraceBatchActivity("other", [event("trace-0")]);
+ await client().zadd(
+ dueKey,
+ ...traces.flatMap((traceId, index) => [
+ Date.now() - 1_000 + index,
+ member("project", traceId),
+ ]),
+ );
+ await makeDue("other", "trace-0");
+ const add = vi.spyOn(queue, "add");
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
+ await trackTraceBatchActivity("disabled", [event("ignored")]);
+ await runner().processBatch();
+ const jobs = await queue.getJobs(["wait"]);
+ for (const job of jobs) TraceBatchEventSchema.parse(job.data);
+ expect(
+ jobs.map((job) => job.data.payload.traces.length).sort((a, b) => a - b),
+ ).toEqual(sizes);
+ expect(
+ jobs
+ .flatMap((job) =>
+ job.data.payload.traces.map((trace) =>
+ member(trace.projectId, trace.traceId),
+ ),
+ )
+ .sort(),
+ ).toEqual(
+ [
+ ...traces.map((traceId) => member("project", traceId)),
+ member("other", "trace-0"),
+ ].sort(),
+ );
+ expect(
+ add.mock.calls.flatMap(([, job]) =>
+ job.payload.traces.map((trace) =>
+ member(trace.projectId, trace.traceId),
+ ),
+ ),
+ ).toEqual([
+ member("other", "trace-0"),
+ ...traces.map((traceId) => member("project", traceId)),
+ ]);
+ expect(await client().zcard(dueKey)).toBe(0);
+ expect(await client().hlen(stateKey)).toBe(0);
+ expect(
+ vi
+ .mocked(recordDistribution)
+ .mock.calls.filter(([name]) => name === "langfuse.trace_batch.size")
+ .map(([, value]) => value)
+ .sort((a, b) => Number(a) - Number(b)),
+ ).toEqual(sizes);
+ expect(
+ vi
+ .mocked(recordDistribution)
+ .mock.calls.filter(
+ ([name]) => name === "langfuse.trace_batch.project_count",
+ )
+ .map(([, value]) => value)
+ .sort((a, b) => Number(a) - Number(b)),
+ ).toEqual(projectCounts);
+ await runner().processBatch();
+ expect(await queue.getWaitingCount()).toBe(sizes.length);
+ },
+ );
+
+ it("groups the complete due cohort by sorted project across hydration chunks and the former run limit", async () => {
+ const due = Date.now() - 100_000;
+ const traces = Array.from({ length: 10_061 }, (_, i) => ({
+ projectId: ["project-c", "project-a", "project-b"][i % 3],
+ traceId: `trace-${String(i).padStart(5, "0")}`,
+ due: due + Math.floor(i / 2_000) * 1_000,
+ }));
+ for (const projectId of ["project-c", "project-a", "project-b"]) {
+ await trackTraceBatchActivity(
+ projectId,
+ traces
+ .filter((trace) => trace.projectId === projectId)
+ .map((trace) => event(trace.traceId)),
+ );
+ }
+ await client().zadd(
+ dueKey,
+ ...traces.flatMap((trace) => [
+ trace.due,
+ member(trace.projectId, trace.traceId),
+ ]),
+ );
+ const add = vi.spyOn(queue, "add");
+ await runner().processBatch();
+ const batches = add.mock.calls.map(([, job]) => job.payload.traces);
+ expect(batches.map((batch) => batch.length)).toEqual([
+ ...Array(167).fill(60),
+ 41,
+ ]);
+ const expected = traces.sort(
+ (a, b) =>
+ a.projectId.localeCompare(b.projectId) ||
+ a.due - b.due ||
+ a.traceId.localeCompare(b.traceId),
+ );
+ expect(
+ batches.flat().map((trace) => member(trace.projectId, trace.traceId)),
+ ).toEqual(expected.map((trace) => member(trace.projectId, trace.traceId)));
+ expect(await client().zcard(dueKey)).toBe(0);
+ expect(await client().hlen(stateKey)).toBe(0);
+ });
+
+ it("revalidates the due ID list after deletion, missing state, reactivation and new arrivals", async () => {
+ const traces = Array.from(
+ { length: 2_001 },
+ (_, i) => `trace-${String(i).padStart(5, "0")}`,
+ );
+ await trackTraceBatchActivity(
+ "project",
+ traces.map((id) => event(id)),
+ );
+ const due = Date.now() - 1_000;
+ await client().zadd(
+ dueKey,
+ ...traces.flatMap((id) => [due, member("project", id)]),
+ );
+ const range = client().zrange.bind(client());
+ let changed = false;
+ vi.spyOn(client(), "zrange").mockImplementation(async (...args) => {
+ const result = await range(...args);
+ if (!changed && args.includes("BYSCORE")) {
+ changed = true;
+ // Change Redis after the complete ID list has been read but before hydration.
+ await client().zrem(dueKey, member("project", traces[0]));
+ await client().hdel(stateKey, member("project", traces[0]));
+ await client().hdel(stateKey, member("project", traces[1_000]));
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 1;
+ await trackTraceBatchActivity("project", [
+ event(traces[999]),
+ event("new-after-cutoff"),
+ ]);
+ }
+ return result;
+ });
+ const add = vi.spyOn(queue, "add");
+ await runner().processBatch();
+ expect(changed).toBe(true);
+ expect(
+ add.mock.calls.flatMap(([, job]) =>
+ job.payload.traces.map((trace) => trace.traceId),
+ ),
+ ).toEqual(traces.filter((_, i) => i !== 0 && i !== 999 && i !== 1_000));
+ expect(
+ await client().zscore(dueKey, member("project", traces[1_000])),
+ ).toBeNull();
+ expect((await client().hkeys(stateKey)).sort()).toEqual(
+ [
+ member("project", "new-after-cutoff"),
+ member("project", traces[999]),
+ ].sort(),
+ );
+ });
+
+ it("expires abandoned pending keys natively without another ingestion or dispatcher run", async () => {
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 300;
+ await trackTraceBatchActivity("project", [event("abandoned")]);
+ for (const key of [dueKey, stateKey]) {
+ expect(await client().pttl(key)).toBeGreaterThan(0);
+ expect(await client().pttl(key)).toBeLessThanOrEqual(300);
+ }
+ await vi.waitFor(async () => {
+ expect(await client().exists(dueKey, stateKey)).toBe(0);
+ });
+ expect(await queue.getWaitingCount()).toBe(0);
+ });
+
+ it("extends native expiry for later activity and never shortens another pending trace's deadline", async () => {
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;
+ await trackTraceBatchActivity("project", [event("long-idle")]);
+ const initialDeadline = Number(await client().pexpiretime(dueKey));
+ await vi.waitFor(() =>
+ expect(Date.now()).toBeGreaterThan(initialDeadline - 60_000 + 10),
+ );
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 20_000;
+ await trackTraceBatchActivity("project", [event("later")]);
+ const extendedDeadline = Number(await client().pexpiretime(dueKey));
+ expect(extendedDeadline).toBeGreaterThan(initialDeadline);
+ expect(Number(await client().pexpiretime(stateKey))).toBe(extendedDeadline);
+
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 1_000;
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 1_000;
+ await trackTraceBatchActivity("project", [event("short-idle")]);
+ expect(Number(await client().pexpiretime(dueKey))).toBeGreaterThanOrEqual(
+ extendedDeadline,
+ );
+ expect(Number(await client().pexpiretime(stateKey))).toBe(
+ Number(await client().pexpiretime(dueKey)),
+ );
+ expect(await client().zcard(dueKey)).toBe(3);
+ expect(await client().hlen(stateKey)).toBe(3);
+ });
+
+ it("drains and removes queued jobs without queries when reads are disabled", async () => {
+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "false";
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
+ const read = vi
+ .spyOn(shared, "getTraceBatchEventStream")
+ .mockImplementation(async function* () {
+ throw new Error("Drain mode must not query ClickHouse");
+ });
+ const payload = {
+ traces: [
+ {
+ projectId: "project",
+ traceId: "trace",
+ minStart: 0,
+ maxStart: 1,
+ revision: "r",
+ },
+ ],
+ };
+ const current = await queue.add(
+ QueueJobs.TraceBatch,
+ {
+ id: "current",
+ timestamp: new Date(),
+ name: QueueJobs.TraceBatch,
+ payload,
+ },
+ { jobId: "current", removeOnComplete: { age: 3600, count: 10000 } },
+ );
+ const old = await queue.add(
+ QueueJobs.TraceBatch,
+ {
+ id: "old",
+ timestamp: new Date(Date.now() - 3 * 60 * 60_000),
+ name: QueueJobs.TraceBatch,
+ payload,
+ },
+ { jobId: "old", removeOnComplete: { age: 3600, count: 10000 } },
+ );
+ const worker = new Worker(queue.name, traceBatchQueueProcessor, {
+ connection,
+ prefix: getQueuePrefix(queue.name),
+ });
+ try {
+ await vi.waitFor(async () => {
+ expect(await queue.getJob(current.id!)).toBeUndefined();
+ expect(await queue.getJob(old.id!)).toBeUndefined();
+ });
+ expect(
+ await queue.getJobCounts("wait", "active", "completed", "failed"),
+ ).toEqual({ wait: 0, active: 0, completed: 0, failed: 0 });
+ expect(read).not.toHaveBeenCalled();
+ } finally {
+ await worker.close();
+ }
+ });
+
+ it("expires bounded pending state during ingestion without dispatch and preserves refreshed traces", async () => {
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;
+ const expired = Array.from({ length: 1_001 }, (_, i) =>
+ event(`expired-${i}`),
+ );
+ await trackTraceBatchActivity("project", [
+ ...expired,
+ event("refreshed"),
+ event("recent"),
+ ]);
+ const cutoff = Date.now() - 70_000;
+ await client().zadd(
+ dueKey,
+ ...expired.flatMap(({ traceId }) => [cutoff, member("project", traceId)]),
+ cutoff,
+ member("project", "refreshed"),
+ );
+ await makeDue("project", "recent");
+ await trackTraceBatchActivity("project", [event("refreshed", 2_000_000)]);
+ expect(await client().zcard(dueKey)).toBe(3);
+ expect(await client().hlen(stateKey)).toBe(3);
+ expect(
+ await client().hget(stateKey, member("project", "recent")),
+ ).not.toBeNull();
+ expect(
+ Number(await client().zscore(dueKey, member("project", "refreshed"))),
+ ).toBeGreaterThan(Date.now());
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.expired_traces",
+ 1_000,
+ );
+ await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);
+ expect((await client().hkeys(stateKey)).sort()).toEqual([
+ member("project", "recent"),
+ member("project", "refreshed"),
+ ]);
+ expect(await client().zcard(dueKey)).toBe(2);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.expired_traces",
+ 1,
+ );
+ expect(await queue.getWaitingCount()).toBe(0);
+ });
+
+ it("cleans an expired backlog in bounded pages without queueing expired traces", async () => {
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;
+ const expired = Array.from({ length: 10_001 }, (_, i) =>
+ event(`expired-${i}`),
+ );
+ await trackTraceBatchActivity("project", [...expired, event("ready")]);
+ await client().zadd(
+ dueKey,
+ ...expired.flatMap(({ traceId }) => [
+ Date.now() - 70_000,
+ member("project", traceId),
+ ]),
+ );
+ await makeDue("project", "ready");
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
+ const dispatcher = runner();
+ await dispatcher.processBatch();
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(
+ ([name]) => name === "langfuse.trace_batch.expired_traces",
+ )
+ .every(([, count]) => count <= 1_000),
+ ).toBe(true);
+ const jobs = await queue.getJobs(["wait"]);
+ expect(
+ jobs.flatMap((job) =>
+ job.data.payload.traces.map((trace) => trace.traceId),
+ ),
+ ).toEqual(["ready"]);
+ expect(await client().hlen(stateKey)).toBe(0);
+ expect(await client().zcard(dueKey)).toBe(0);
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(
+ ([name]) => name === "langfuse.trace_batch.expired_traces",
+ )
+ .reduce((sum, [, value]) => sum + value, 0),
+ ).toBe(10_001);
+ });
+
+ it("keeps due state on enqueue failure and retries it on the next dispatch", async () => {
+ await trackTraceBatchActivity("project", [event("trace")]);
+ await makeDue("project", "trace");
+ vi.spyOn(queue, "add").mockRejectedValueOnce(
+ new Error("enqueue unavailable"),
+ );
+ const dispatcher = runner();
+ await dispatcher.processBatch();
+ expect(await client().zcard(dueKey)).toBe(1);
+ expect(await client().hlen(stateKey)).toBe(1);
+ await dispatcher.processBatch();
+ expect(await queue.getWaitingCount()).toBe(1);
+ expect(await client().zcard(dueKey)).toBe(0);
+ });
+
+ it.each(["updated", "deleted", "expired"] as const)(
+ "preserves arrivals during enqueue, including deleted/recreated state: %s",
+ async (mode) => {
+ await trackTraceBatchActivity("project", [event("trace")]);
+ await makeDue("project", "trace");
+ const add = queue.add.bind(queue);
+ vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
+ const job = await add(...args);
+ if (mode === "deleted")
+ await client().hdel(stateKey, member("project", "trace"));
+ if (mode === "expired") {
+ await client().zadd(dueKey, 0, member("project", "trace"));
+ await trackTraceBatchActivity("other", [event("trigger")]);
+ expect(
+ await client().hget(stateKey, member("project", "trace")),
+ ).toBeNull();
+ await client().zrem(dueKey, member("other", "trigger"));
+ await client().hdel(stateKey, member("other", "trigger"));
+ }
+ await trackTraceBatchActivity("project", [event("trace", 2_000_000)]);
+ return job;
+ });
+ const dispatcher = runner();
+ await dispatcher.processBatch();
+ expect(await queue.getWaitingCount()).toBe(1);
+ expect(await client().zcard(dueKey)).toBe(1);
+ const current = JSON.parse(
+ (await client().hget(stateKey, member("project", "trace")))!,
+ );
+ const [job] = await queue.getJobs(["wait"]);
+ expect(current.revision).not.toBe(job.data.payload.traces[0].revision);
+ expect(current.maxStart).toBe(2_000_000);
+ await makeDue("project", "trace");
+ await dispatcher.processBatch();
+ expect(await queue.getWaitingCount()).toBe(2);
+ expect(await client().hlen(stateKey)).toBe(0);
+ },
+ );
+
+ it("reuses a queued batch when its acknowledgement fails", async () => {
+ await trackTraceBatchActivity("project", [event("trace")]);
+ await makeDue("project", "trace");
+ await trackTraceBatchActivity("other", [event("trace")]);
+ await makeDue("other", "trace");
+ const add = queue.add.bind(queue);
+ vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
+ const job = await add(...args);
+ // The next Redis operation is the post-enqueue acknowledgement.
+ vi.spyOn(client(), "eval").mockRejectedValueOnce(
+ new Error("ACK unavailable"),
+ );
+ return job;
+ });
+ const dispatcher = runner();
+ await dispatcher.processBatch();
+ expect(await queue.getWaitingCount()).toBe(1);
+ expect(await client().hlen(stateKey)).toBe(2);
+ await dispatcher.processBatch();
+ expect(await queue.getWaitingCount()).toBe(1);
+ expect(await client().hlen(stateKey)).toBe(0);
+ const [job] = await queue.getJobs(["wait"]);
+ expect(
+ job.data.payload.traces
+ .map((trace) => member(trace.projectId, trace.traceId))
+ .sort(),
+ ).toEqual([member("other", "trace"), member("project", "trace")]);
+ });
+
+ it("subtracts dispatcher runtime from the next delay and catches up after an overrun", async () => {
+ let clock = Date.now();
+ const interval = env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;
+ let elapsed = Math.floor(interval / 2);
+ vi.spyOn(Date, "now").mockImplementation(() => clock);
+ const range = client().zrange.bind(client());
+ vi.spyOn(client(), "zrange").mockImplementation(async (...args) => {
+ const result = await range(...args);
+ if (args.includes("BYSCORE")) clock += elapsed;
+ return result;
+ });
+ const dispatcher = runner();
+ expect(await dispatcher.processBatch()).toBe(interval - elapsed);
+ elapsed = interval + 1_000;
+ expect(await dispatcher.processBatch()).toBe(0);
+ });
+
+ it("bounds shutdown when an in-flight enqueue stalls and remains stopped after recovery", async () => {
+ await trackTraceBatchActivity("project", [event("trace")]);
+ await makeDue("project", "trace");
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const add = queue.add.bind(queue);
+ vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
+ entered.resolve();
+ await release.promise;
+ return add(...args);
+ });
+ const dispatcher = runner();
+ const active = dispatcher.processBatch();
+ await entered.promise;
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ let drained = false;
+ const drain = dispatcher.drain().then(() => {
+ drained = true;
+ });
+ try {
+ await vi.advanceTimersByTimeAsync(4_999);
+ expect(drained).toBe(false);
+ await vi.advanceTimersByTimeAsync(1);
+ expect(drained).toBe(true);
+ } finally {
+ vi.useRealTimers();
+ release.resolve();
+ await Promise.all([active, drain]);
+ }
+ await trackTraceBatchActivity("project", [event("next")]);
+ await makeDue("project", "next");
+ await dispatcher.processBatch();
+ expect(queue.add).toHaveBeenCalledTimes(1);
+ expect(await client().hlen(stateKey)).toBe(1);
+ });
+
+ it("allows only one dispatcher and waits for its in-flight enqueue before shutdown", async () => {
+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 1;
+ const traces = ["trace"];
+ await trackTraceBatchActivity(
+ "project",
+ traces.map((traceId) => event(traceId)),
+ );
+ await makeDue("project", ...traces);
+ await trackTraceBatchActivity("other", [event("trace")]);
+ await makeDue("other", "trace");
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const add = queue.add.bind(queue);
+ vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
+ entered.resolve();
+ await release.promise;
+ return add(...args);
+ });
+ const first = runner();
+ const active = first.processBatch();
+ await entered.promise;
+ await runner().processBatch();
+ expect(queue.add).toHaveBeenCalledTimes(1);
+ let drained = false;
+ const drain = first.drain().then(() => {
+ drained = true;
+ });
+ await Promise.resolve();
+ expect(drained).toBe(false);
+ release.resolve();
+ await Promise.all([active, drain]);
+ expect(drained).toBe(true);
+ expect(await queue.getWaitingCount()).toBe(1);
+ expect(await client().hlen(stateKey)).toBe(1);
+ await first.processBatch();
+ expect(queue.add).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/worker/src/app.ts b/worker/src/app.ts
index 3563a43b70..f979e0e4c2 100644
--- a/worker/src/app.ts
+++ b/worker/src/app.ts
@@ -58,6 +58,8 @@ import { prisma } from "@langfuse/shared/src/db";
import { ClickhouseReadSkipCache } from "./utils/clickhouseReadSkipCache";
import { experimentCreateQueueProcessor } from "./queues/experimentQueue";
import { traceDeleteProcessor } from "./queues/traceDelete";
+import { traceBatchQueueProcessor } from "./queues/traceBatchQueue";
+import { TraceBatchDispatcher } from "./features/traceBatching/traceBatching";
import { projectDeleteProcessor } from "./queues/projectDelete";
import {
postHogIntegrationProcessingProcessor,
@@ -136,6 +138,24 @@ ClickhouseReadSkipCache.getInstance(prisma)
logger.error("Error initializing ClickhouseReadSkipCache", err);
});
+export let traceBatchDispatcher: TraceBatchDispatcher | null = null;
+if (
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED === "true"
+) {
+ traceBatchDispatcher = new TraceBatchDispatcher();
+ traceBatchDispatcher.start();
+}
+
+if (
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED === "true"
+) {
+ WorkerManager.register(QueueName.TraceBatch, traceBatchQueueProcessor, {
+ concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,
+ });
+}
+
if (env.QUEUE_CONSUMER_TRACE_UPSERT_QUEUE_IS_ENABLED === "true") {
// Register workers for all trace upsert queue shards
const traceUpsertShardNames = TraceUpsertQueue.getShardNames();
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 4cae45be30..ac19d4a74c 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -106,6 +106,47 @@ const EnvSchema = z.object({
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: z
.enum(["true", "false"])
.default("false"),
+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: z
+ .enum(["true", "false"])
+ .default("false"),
+ LANGFUSE_TRACE_BATCH_SAMPLING_RATE: z.coerce
+ .number()
+ .min(0)
+ .max(1)
+ .default(1),
+ LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED: z
+ .enum(["true", "false"])
+ .default("false"),
+ QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED: z
+ .enum(["true", "false"])
+ .default("false"),
+ LANGFUSE_TRACE_BATCH_READ_ENABLED: z.enum(["true", "false"]).default("false"),
+ LANGFUSE_TRACE_BATCH_CONCURRENCY: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(2),
+ LANGFUSE_TRACE_BATCH_MAX_SIZE: z.coerce
+ .number()
+ .int()
+ .positive()
+ .max(1_000)
+ .default(60),
+ LANGFUSE_TRACE_BATCH_IDLE_MS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(600_000),
+ LANGFUSE_TRACE_BATCH_PENDING_TTL_MS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(7_200_000),
+ LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(30_000),
LANGFUSE_SECONDARY_OTEL_INGESTION_QUEUE_ENABLED_PROJECT_IDS: z
.string()
.optional(),
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
new file mode 100644
index 0000000000..e8e3c23207
--- /dev/null
+++ b/worker/src/features/traceBatching/README.md
@@ -0,0 +1,117 @@
+# Internal cloud trace-batch experiment
+
+This read-only experiment measures full-event reads for idle traces. It requires
+`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` and explicit opt-in. All four enablement
+flags below default to `false`; an ordinary release needs no infrastructure
+changes. These internal controls are intentionally absent from env templates.
+
+## Controls
+
+| Variable | Default | Role |
+| --------------------------------------------- | --------- | ------------------------------------------------------------- |
+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |
+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 1,000. |
+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |
+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
+
+Producer, dispatcher and consumer can run independently. Sampling reuses the
+evaluator's deterministic trace-ID sampler; every observation of the same trace
+gets the same admission decision. Dispatch and consumption do not resample.
+Cloud guards cover ingestion, dispatcher execution, worker registration and
+processing, and the shared queue accessor, including admin queue inspection.
+
+## Redis state and expiry
+
+- `{trace-batch}:due`: sorted set, with JSON `[projectId, traceId]` members and
+ Redis server time plus idle delay as the score.
+- `{trace-batch}:state`: hash with the same members and JSON
+ `{minStart, maxStart, revision}` values. Bounds describe observed event start
+ times; revision protects newer ingestion from an older dispatch acknowledgment.
+- `{trace-batch}:dispatcher`: renewable dispatcher lease.
+
+No event payloads, per-trace byte counts or observation counts are stored here.
+Atomic Lua updates refresh bounds, revision and readiness. Each admitted intake
+sets a shared absolute expiry on both pending keys with `PEXPIREAT`, preserving
+any later existing expiry using `PEXPIRETIME` (Redis 7+). With no further intake,
+both keys expire after the retention period even if every application stops;
+idle time is not added to this deadline.
+
+This is a whole-key inactivity backstop, not native per-member expiry. During
+continuous intake, bounded Lua cleanup removes members whose due time is older
+than retention; ingestion and dispatcher both perform cleanup.
+
+## Dispatch and reads
+
+One dispatcher holds the renewable lease. It snapshots ready IDs, sorts them by
+project, then hydrates and revalidates state in chunks of 1,000. Jobs contain up
+to the configured trace cap; project leftovers can share a job. The dispatcher
+enqueues before acknowledging matching revisions, so concurrent intake stays
+pending and failed enqueueing can be retried. Stable job IDs limit duplicates.
+
+The worker streams full `events_full` payloads and records batch observation,
+trace, project and logical payload-byte metrics. The `input_bytes`, `output_bytes`
+and `metadata_bytes` distributions under `langfuse.trace_batch` each report one
+total per completed batch read; `io_metadata_bytes` retains their combined total.
+Sizes count UTF-8 bytes, including metadata keys and values, excluding JSON
+transport framing/escaping and compression. They do not measure ClickHouse bytes
+scanned or per-trace ingestion size; retries can record another batch sample.
+Exact `(projectId, traceId)`
+pairs protect tenant boundaries; project, trace hash and a shared batch time
+window prune reads. `TRACE_QUERY_BUFFER_MS` pads the earliest and latest recorded
+start times by two minutes each. This fixed query margin can include observations
+outside the recorded bounds; it neither waits for arrivals nor guarantees trace
+completeness. Query limits are fixed at two threads and 30 seconds with
+timeouts failing the job. This baseline has no hash-locality grouping,
+per-trace windows, query tuning controls or allocation based on trace size.
+
+## Monitor dispatcher memory
+
+The baseline intentionally snapshots the complete ready cohort. Batch size limits
+jobs, not snapshot memory. Retain this behavior for the initial experiment and
+watch memory alongside backlog growth before deciding whether to add paging.
+
+The worker enables Datadog Node runtime metrics. In Metrics Explorer, filter to
+the experiment environment and worker service, and inspect the maximum per
+worker/container rather than a fleet average:
+
+- `runtime.node.mem.rss`: total process memory; compare with the container limit.
+- `runtime.node.mem.heap_used`: JavaScript heap in use.
+- `langfuse.trace_batch.snapshot_size`, `pending_traces`, `ready_traces` and
+ `oldest_due_age_ms`: snapshot and backlog growth.
+- `langfuse.periodic_runner.duration_ms` and `completed`, filtered by
+ `runner:trace_batch_dispatcher`: run duration and outcome.
+
+Runtime memory is shared with other work in the process, so correlation with
+dispatch runs does not isolate the dispatcher's allocations. Periodic sampling
+can miss short peaks, and `snapshot_size` is emitted only after parsing/sorting
+finishes. Check worker restarts/OOM events as well. Verify live metric delivery
+and available instance tags in Datadog before relying on these signals.
+
+## Stop and drain
+
+Disable ingestion and dispatch to stop new work. Leave the consumer enabled and
+set the read flag to `false` to complete and remove jobs without querying or
+parsing their payloads. Jobs at least two hours old are also discarded on pickup.
+Reads already in progress finish under their query timeout.
+
+On process shutdown, the dispatcher stops scheduling and waits up to five seconds
+for its active dispatch. If Redis stalls, it logs a warning and lets the remaining
+shutdown steps proceed. This timeout does not cancel an already issued Redis command.
+
+BullMQ waiting jobs have no autonomous TTL: disabling all consumers leaves them
+queued until consumption resumes. Draining requires an active, unpaused worker;
+delayed jobs become eligible later. Normal completed jobs retain IDs for up to
+one hour / 10,000 jobs, and failures retain up to 1,000 jobs under BullMQ's
+lazy retention rules. Drain and expired jobs are removed immediately on success.
+
+For an already enabled cloud experiment, explicitly set the new read flag to
+`true` when deploying this version if reads should continue; leaving it unset
+drains instead. Upgrade all producers before relying on native expiry: older
+producers do not install or refresh that TTL. Locality, query controls and load
+test artifacts belong in separate follow-up changes.
diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.ts
new file mode 100644
index 0000000000..a2dfa70155
--- /dev/null
+++ b/worker/src/features/traceBatching/traceBatching.ts
@@ -0,0 +1,409 @@
+import { createHash, randomUUID } from "node:crypto";
+import {
+ logger,
+ QueueJobs,
+ redis,
+ recordDistribution,
+ recordGauge,
+ recordIncrement,
+ TraceBatchTraceSchema,
+ TraceBatchQueue,
+ type QueueName,
+ type TQueueJobTypes,
+} from "@langfuse/shared/src/server";
+import { env } from "../../env";
+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";
+import {
+ getDeterministicSamplingValue,
+ shouldSampleEvaluation,
+} from "../evaluation/deterministicSampling";
+
+// The shared client applies REDIS_KEY_PREFIX. Both keys must occupy one
+// Redis Cluster slot for the atomic update, snapshot, and acknowledgement.
+const DUE_KEY = "{trace-batch}:due";
+const STATE_KEY = "{trace-batch}:state";
+const CHUNK_SIZE = 1_000;
+type PendingTrace = {
+ member: string;
+ due: number;
+ trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];
+};
+
+// Retain traces for a bounded time after readiness. Reuse the due index so
+// cleanup needs no full hash scan. Whole-key expiry is an inactivity backstop.
+const EXPIRE_PENDING_SCRIPT = `
+ local function expirePending(now, retention, limit)
+ local expired = redis.call('ZRANGE', KEYS[1], '-inf', now - retention,
+ 'BYSCORE', 'LIMIT', 0, limit)
+ for _, member in ipairs(expired) do
+ redis.call('ZREM', KEYS[1], member)
+ redis.call('HDEL', KEYS[2], member)
+ end
+ return #expired
+ end
+`;
+
+const TRACK_SCRIPT = `
+ ${EXPIRE_PENDING_SCRIPT}
+ local clock = redis.call('TIME')
+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)
+ for i = 3, #ARGV, 4 do
+ local member = ARGV[i]
+ local minStart = tonumber(ARGV[i + 1])
+ local maxStart = tonumber(ARGV[i + 2])
+ local previous = redis.call('HGET', KEYS[2], member)
+ if previous then
+ local state = cjson.decode(previous)
+ minStart = math.min(minStart, state.minStart)
+ maxStart = math.max(maxStart, state.maxStart)
+ end
+ redis.call('HSET', KEYS[2], member, cjson.encode({
+ minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3]
+ }))
+ redis.call('ZADD', KEYS[1], now + tonumber(ARGV[1]), member)
+ end
+ local retention = tonumber(ARGV[2])
+ local expired = expirePending(now, retention, ${CHUNK_SIZE})
+ -- Bound abandoned state from the last admitted activity, independent of idle.
+ -- A shorter policy on another writer must not shorten an existing expiry.
+ local expiresAt = math.max(now + retention,
+ redis.call('PEXPIRETIME', KEYS[1]), redis.call('PEXPIRETIME', KEYS[2]))
+ redis.call('PEXPIREAT', KEYS[1], expiresAt)
+ redis.call('PEXPIREAT', KEYS[2], expiresAt)
+ return expired
+`;
+
+// Cleanup stays bounded even when the dispatcher has accumulated a backlog.
+// Keep the first Redis timestamp as the cutoff throughout this run.
+const SNAPSHOT_SCRIPT = `
+ ${EXPIRE_PENDING_SCRIPT}
+ local clock = redis.call('TIME')
+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)
+ local retention = tonumber(ARGV[1])
+ local cutoff = tonumber(ARGV[2]) or now
+ local expired = expirePending(now, retention, ${CHUNK_SIZE})
+ local pending = redis.call('ZCARD', KEYS[1])
+ local ready = math.max(0, redis.call('ZCOUNT', KEYS[1], '-inf', cutoff)
+ - redis.call('ZCOUNT', KEYS[1], '-inf', now - retention))
+ local first = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')
+ return {now, cutoff, pending, ready, first[2] or tostring(now), expired}
+`;
+
+// Fetch state only while it is still due for this run. Checking the score and
+// reading the revision atomically prevents dispatching freshly reactivated work.
+const HYDRATE_SCRIPT = `
+ local clock = redis.call('TIME')
+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)
+ local result = {}
+ for i = 3, #ARGV do
+ local member = ARGV[i]
+ local due = tonumber(redis.call('ZSCORE', KEYS[1], member))
+ if due and due <= tonumber(ARGV[1]) and due > now - tonumber(ARGV[2]) then
+ local state = redis.call('HGET', KEYS[2], member)
+ if state then
+ table.insert(result, member)
+ table.insert(result, state)
+ table.insert(result, tostring(due))
+ else
+ redis.call('ZREM', KEYS[1], member)
+ end
+ end
+ end
+ return result
+`;
+
+const ACKNOWLEDGE_SCRIPT = `
+ local removed = 0
+ for i = 1, #ARGV, 2 do
+ local state = redis.call('HGET', KEYS[2], ARGV[i])
+ if state and cjson.decode(state).revision == ARGV[i + 1] then
+ redis.call('ZREM', KEYS[1], ARGV[i])
+ redis.call('HDEL', KEYS[2], ARGV[i])
+ removed = removed + 1
+ end
+ end
+ return removed
+`;
+
+export async function trackTraceBatchActivity(
+ projectId: string,
+ events: { traceId: string; startTimeISO: string }[],
+): Promise<void> {
+ if (
+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED !== "true"
+ )
+ return;
+
+ const bounds = new Map<string, { minStart: number; maxStart: number }>();
+ for (const event of events) {
+ const start = Date.parse(event.startTimeISO);
+ if (!Number.isFinite(start)) continue;
+ const previous = bounds.get(event.traceId);
+ bounds.set(event.traceId, {
+ minStart: Math.min(previous?.minStart ?? start, start),
+ maxStart: Math.max(previous?.maxStart ?? start, start),
+ });
+ }
+ if (bounds.size === 0) return;
+
+ try {
+ const samplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;
+ // Sample by trace ID so later observations and retries keep the same decision.
+ // Dispatcher and consumer process admitted work without resampling.
+ const entries = [...bounds].filter(
+ ([traceId]) =>
+ samplingRate === 1 ||
+ (samplingRate > 0 &&
+ shouldSampleEvaluation({
+ samplingValue: getDeterministicSamplingValue(traceId),
+ samplingRate,
+ })),
+ );
+ recordGauge("langfuse.trace_batch.sampling_rate", samplingRate);
+ // Counts distinct trace IDs per ingestion batch, not globally unique traces.
+ recordIncrement("langfuse.trace_batch.sampling_decisions", entries.length, {
+ decision: "selected",
+ });
+ recordIncrement(
+ "langfuse.trace_batch.sampling_decisions",
+ bounds.size - entries.length,
+ { decision: "excluded" },
+ );
+ if (entries.length === 0) return;
+ if (!redis) throw new Error("Trace batching requires Redis");
+ // Bounded scripts keep ingestion from monopolizing the global Redis slot.
+ for (let offset = 0; offset < entries.length; offset += CHUNK_SIZE) {
+ const chunk = entries.slice(offset, offset + CHUNK_SIZE);
+ const expired = Number(
+ await redis.eval(
+ TRACK_SCRIPT,
+ 2,
+ DUE_KEY,
+ STATE_KEY,
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS,
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
+ ...chunk.flatMap(([traceId, state]) => [
+ JSON.stringify([projectId, traceId]),
+ state.minStart,
+ state.maxStart,
+ // Unique even after dispatch deletes and a later arrival recreates state.
+ randomUUID(),
+ ]),
+ ),
+ );
+ recordIncrement("langfuse.trace_batch.expired_traces", expired);
+ recordIncrement("langfuse.trace_batch.tracked_traces", chunk.length);
+ }
+ } catch (error) {
+ // This optional load experiment must not fail normal event ingestion.
+ recordIncrement("langfuse.trace_batch.tracking_errors", 1);
+ logger.error("Failed to track trace batch activity", { error });
+ }
+}
+
+export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
+ private stopping = false;
+ private activeDispatch: Promise<void> | null = null;
+
+ constructor() {
+ super({
+ name: "TraceBatchDispatcher",
+ metricName: "trace_batch_dispatcher",
+ lockKey: "{trace-batch}:dispatcher",
+ lockTtlSeconds: 60,
+ onUnavailable: "fail",
+ });
+ }
+
+ protected get defaultIntervalMs(): number {
+ return env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;
+ }
+
+ public override stop(): void {
+ this.stopping = true;
+ super.stop();
+ }
+
+ public async drain(): Promise<void> {
+ this.stop();
+ if (!this.activeDispatch) return;
+ let timeout: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ this.activeDispatch,
+ new Promise<void>((resolve) => {
+ timeout = setTimeout(() => {
+ logger.warn(
+ "Trace batch dispatcher drain timed out after 5 seconds; continuing shutdown",
+ );
+ resolve();
+ }, 5_000);
+ }),
+ ]);
+ } finally {
+ clearTimeout(timeout);
+ }
+ }
+
+ protected async execute(): Promise<number | void> {
+ if (
+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED !== "true" ||
+ this.stopping ||
+ this.activeDispatch
+ )
+ return;
+ const startedAt = Date.now();
+ const dispatch = this.withLock(() => this.dispatchDue()).then(() => {});
+ this.activeDispatch = dispatch;
+ try {
+ await dispatch;
+ } finally {
+ this.activeDispatch = null;
+ }
+ // PeriodicRunner waits this delay after completion. Account for work time
+ // to maintain the configured start cadence without overlapping runs.
+ return Math.max(0, this.defaultIntervalMs - (Date.now() - startedAt));
+ }
+
+ private async dispatchDue(): Promise<void> {
+ if (!redis) throw new Error("Trace batching requires Redis");
+ const queue = TraceBatchQueue.getInstance();
+ if (!queue) throw new Error("Trace batch queue is unavailable");
+ let cutoff = "";
+ let now = 0;
+ while (!this.stopping) {
+ await this.extendLockOnProgress();
+ const snapshot = (await redis.eval(
+ SNAPSHOT_SCRIPT,
+ 2,
+ DUE_KEY,
+ STATE_KEY,
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
+ cutoff,
+ )) as (string | number)[];
+ const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);
+ now = clock;
+ cutoff = String(snapshot[1]);
+ recordIncrement("langfuse.trace_batch.expired_traces", expired);
+ recordGauge("langfuse.trace_batch.pending_traces", pending);
+ recordGauge("langfuse.trace_batch.ready_traces", ready);
+ recordGauge(
+ "langfuse.trace_batch.oldest_due_age_ms",
+ Math.max(0, now - oldest),
+ );
+ if (expired < CHUNK_SIZE) break;
+ }
+ if (this.stopping) return;
+
+ // One range read captures all due IDs without cursor races or loading every
+ // trace's state. Memory and the Redis response size grow with the due cohort.
+ const members = (
+ await redis.zrange(
+ DUE_KEY,
+ `(${now - env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS}`,
+ cutoff,
+ "BYSCORE",
+ )
+ ).map((member) => ({
+ member,
+ projectId: (JSON.parse(member) as [string, string])[0],
+ }));
+ // Stable sorting keeps Redis readiness order within each project.
+ members.sort((a, b) =>
+ a.projectId < b.projectId ? -1 : a.projectId > b.projectId ? 1 : 0,
+ );
+ recordDistribution("langfuse.trace_batch.snapshot_size", members.length);
+
+ const enqueue = async (batch: PendingTrace[]) => {
+ await this.extendLockOnProgress(true);
+ if (this.stopping) return;
+ const traces = batch.map(({ trace }) => trace);
+ const id = createHash("sha256")
+ .update(JSON.stringify(traces))
+ .digest("hex");
+ // Stable IDs reduce duplicate delivery if enqueue succeeds but ACK fails.
+ // Reads remain retry-safe even if regrouping produces a different job ID.
+ await queue.add(
+ QueueJobs.TraceBatch,
+ {
+ id,
+ timestamp: new Date(),
+ name: QueueJobs.TraceBatch,
+ payload: { traces },
+ },
+ { jobId: id },
+ );
+ recordDistribution("langfuse.trace_batch.size", batch.length);
+ recordDistribution(
+ "langfuse.trace_batch.project_count",
+ new Set(traces.map((trace) => trace.projectId)).size,
+ );
+ recordIncrement("langfuse.trace_batch.dispatched_traces", batch.length, {
+ batch_kind: batch.length === 1 ? "singleton" : "multi",
+ });
+ for (const entry of batch) {
+ recordDistribution(
+ "langfuse.trace_batch.due_lag_ms",
+ Math.max(0, Date.now() - entry.due),
+ );
+ }
+ const removed = Number(
+ await redis!.eval(
+ ACKNOWLEDGE_SCRIPT,
+ 2,
+ DUE_KEY,
+ STATE_KEY,
+ ...batch.flatMap(({ member, trace }) => [member, trace.revision]),
+ ),
+ );
+ recordIncrement(
+ "langfuse.trace_batch.reactivated_traces",
+ batch.length - removed,
+ );
+ };
+
+ let batch: PendingTrace[] = [];
+ for (
+ let offset = 0;
+ offset < members.length && !this.stopping;
+ offset += CHUNK_SIZE
+ ) {
+ await this.extendLockOnProgress();
+ const candidates = members.slice(offset, offset + CHUNK_SIZE);
+ const hydrated = (await redis.eval(
+ HYDRATE_SCRIPT,
+ 2,
+ DUE_KEY,
+ STATE_KEY,
+ cutoff,
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
+ ...candidates.map(({ member }) => member),
+ )) as string[];
+ recordIncrement(
+ "langfuse.trace_batch.skipped_traces",
+ candidates.length - hydrated.length / 3,
+ );
+ for (let i = 0; i < hydrated.length; i += 3) {
+ if (this.stopping) return;
+ const member = hydrated[i];
+ const [projectId, traceId] = JSON.parse(member) as [string, string];
+ batch.push({
+ member,
+ due: Number(hydrated[i + 2]),
+ trace: TraceBatchTraceSchema.parse({
+ ...JSON.parse(hydrated[i + 1]),
+ projectId,
+ traceId,
+ }),
+ });
+ if (batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {
+ await enqueue(batch);
+ batch = [];
+ }
+ }
+ }
+ if (!this.stopping && batch.length > 0) await enqueue(batch);
+ }
+}
diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
index 7a91be20b8..1b65d2ce64 100644
--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
@@ -1,4 +1,18 @@
-import { describe, it, expect } from "vitest";
+import { describe, it, expect, vi } from "vitest";
+import { type Job } from "bullmq";
+import {
+ getS3EventStorageClient,
+ OtelIngestionProcessor,
+ QueueJobs,
+ type QueueName,
+ recordDistribution,
+ type TQueueJobTypes,
+} from "@langfuse/shared/src/server";
+import { env } from "../../env";
+import { IngestionService } from "../../services/IngestionService";
+import { ClickhouseWriter } from "../../services/ClickhouseWriter";
+import { fetchObservationEvalRules } from "../../features/evaluation/observationEval";
+import { trackTraceBatchActivity } from "../../features/traceBatching/traceBatching";
import {
batchContainsLangfuseScope,
checkHeaderBasedDirectWrite,
@@ -6,11 +20,148 @@ import {
getSdkInfoFromResourceSpans,
isLangfuseSdkTraffic,
isOrgPastOtelDirectWriteCutoff,
+ otelIngestionQueueProcessorBuilder,
resolveOtelWritePath,
shouldProcessLegacyOtelMedia,
type SdkInfo,
} from "../otelIngestionQueue";
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
+ getS3EventStorageClient: vi.fn(),
+ recordDistribution: vi.fn(),
+}));
+vi.mock(
+ "../../features/evaluation/observationEval",
+ async (importOriginal) => ({
+ ...(await importOriginal<
+ typeof import("../../features/evaluation/observationEval")
+ >()),
+ fetchObservationEvalRules: vi.fn(),
+ }),
+);
+vi.mock("../../features/traceBatching/traceBatching", () => ({
+ trackTraceBatchActivity: vi.fn(),
+}));
+
+describe("direct-v4 trace batch tracking", () => {
+ it("tracks accepted writes together and excludes failed writes, disabled intake, and legacy writes", async () => {
+ const original = {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED:
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED,
+ LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,
+ LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED:
+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,
+ };
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED = "false";
+
+ try {
+ vi.mocked(getS3EventStorageClient).mockReturnValue({
+ download: vi.fn().mockResolvedValue("[]"),
+ } as unknown as ReturnType<typeof getS3EventStorageClient>);
+ vi.mocked(fetchObservationEvalRules).mockResolvedValue([]);
+ vi.spyOn(ClickhouseWriter, "getInstance").mockReturnValue(
+ {} as ClickhouseWriter,
+ );
+ vi.spyOn(
+ OtelIngestionProcessor.prototype,
+ "processToIngestionEvents",
+ ).mockResolvedValue([]);
+ const inputs = [
+ { traceId: "accepted", spanId: "first" },
+ { traceId: "accepted", spanId: "second" },
+ { traceId: "write-failed", spanId: "third" },
+ { traceId: "conversion-failed", spanId: "fourth" },
+ ].map((entry, index) => ({
+ ...entry,
+ projectId: "project",
+ startTimeISO: new Date(1_000 + index * 1_000).toISOString(),
+ }));
+ vi.spyOn(
+ OtelIngestionProcessor.prototype,
+ "processToEvent",
+ ).mockReturnValue(
+ inputs as ReturnType<OtelIngestionProcessor["processToEvent"]>,
+ );
+ vi.spyOn(
+ IngestionService.prototype,
+ "createEventRecord",
+ ).mockImplementation(async (input) => {
+ if (input.traceId === "conversion-failed") throw new Error("invalid");
+ return { span_id: input.spanId } as Awaited<
+ ReturnType<IngestionService["createEventRecord"]>
+ >;
+ });
+ const write = vi
+ .spyOn(IngestionService.prototype, "writeEventRecord")
+ .mockImplementation(async (record) => {
+ if (record.span_id === "third") throw new Error("write rejected");
+ });
+ const processor = otelIngestionQueueProcessorBuilder(false);
+ const job = {
+ data: {
+ id: "ingestion",
+ timestamp: new Date(),
+ name: QueueJobs.OtelIngestionJob,
+ payload: {
+ data: { fileKey: "synthetic.json" },
+ authCheck: {
+ validKey: true,
+ scope: {
+ projectId: "project",
+ orgId: "org",
+ accessLevel: "project",
+ },
+ },
+ ingestionVersion: "4",
+ },
+ },
+ } as Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>;
+
+ await processor(job, undefined);
+ expect(write).toHaveBeenCalledTimes(3);
+ expect(trackTraceBatchActivity).toHaveBeenCalledExactlyOnceWith(
+ "project",
+ inputs
+ .slice(0, 2)
+ .map(({ traceId, startTimeISO }) => ({ traceId, startTimeISO })),
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.ingestion_trace_count",
+ 1,
+ );
+
+ vi.mocked(trackTraceBatchActivity).mockClear();
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+ await processor(job, undefined);
+ expect(trackTraceBatchActivity).not.toHaveBeenCalled();
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
+ await processor(job, undefined);
+ expect(trackTraceBatchActivity).not.toHaveBeenCalled();
+
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
+ write.mockRejectedValue(new Error("all writes rejected"));
+ await processor(job, undefined);
+ expect(trackTraceBatchActivity).not.toHaveBeenCalled();
+
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";
+ write.mockClear();
+ await processor(job, undefined);
+ expect(write).not.toHaveBeenCalled();
+ expect(trackTraceBatchActivity).not.toHaveBeenCalled();
+ } finally {
+ Object.assign(env, original);
+ vi.restoreAllMocks();
+ vi.clearAllMocks();
+ }
+ });
+});
+
describe("isOrgPastOtelDirectWriteCutoff", () => {
const cutoff = "2026-08-06";
const inScope = {
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
new file mode 100644
index 0000000000..cdb29bed53
--- /dev/null
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -0,0 +1,254 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { type Job } from "bullmq";
+import {
+ getTraceBatchEventStream,
+ QueueJobs,
+ type QueueName,
+ recordDistribution,
+ type TQueueJobTypes,
+} from "@langfuse/shared/src/server";
+import { env } from "../../env";
+import { traceBatchQueueProcessor } from "../traceBatchQueue";
+
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
+ getTraceBatchEventStream: vi.fn(),
+ recordDistribution: vi.fn(),
+}));
+
+const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
+const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+beforeEach(() => {
+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+});
+afterEach(() => {
+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
+ vi.restoreAllMocks();
+ vi.clearAllMocks();
+});
+
+describe("trace batch queue", () => {
+ it("discards self-hosted jobs before parsing even when reads are enabled", async () => {
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+ const job = {
+ opts: {},
+ data: { incompatiblePayload: true },
+ } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({
+ discarded: "not_cloud",
+ });
+ expect(getTraceBatchEventStream).not.toHaveBeenCalled();
+ expect(recordDistribution).not.toHaveBeenCalled();
+ expect(job.opts.removeOnComplete).toBe(true);
+ });
+
+ it("discards an expired batch before querying, including on retry", async () => {
+ const job = {
+ opts: {},
+ data: {
+ id: "expired",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(Date.now() - 3 * 60 * 60_000).toISOString(),
+ payload: {
+ traces: [
+ {
+ projectId: "project",
+ traceId: "trace",
+ minStart: 0,
+ maxStart: 1,
+ revision: "r",
+ },
+ ],
+ },
+ },
+ } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+ for (let attempt = 0; attempt < 2; attempt++) {
+ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({
+ discarded: "expired",
+ });
+ }
+ expect(getTraceBatchEventStream).not.toHaveBeenCalled();
+ expect(recordDistribution).not.toHaveBeenCalled();
+ expect(job.opts.removeOnComplete).toBe(true);
+ });
+
+ it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {
+ const ingestionEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;
+ const dispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "false";
+ try {
+ const payload = {
+ traces: [
+ ["project", "trace-a"],
+ ["project", "trace-b"],
+ ["other", "trace-a"],
+ ["project", "trace-missing"],
+ ["absent", "trace-missing"],
+ ].map(([projectId, traceId]) => ({
+ projectId,
+ traceId,
+ minStart: 1_000,
+ maxStart: 2_000,
+ revision: "revision",
+ })),
+ };
+ const job = {
+ data: {
+ id: "batch",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(),
+ payload,
+ },
+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+ let yieldedRows = 0;
+ vi.mocked(getTraceBatchEventStream).mockImplementation(
+ async function* () {
+ for (const [projectId, traceId] of [
+ ["project", "trace-a"],
+ ["project", "trace-a"],
+ ["project", "trace-b"],
+ ["other", "trace-a"],
+ ]) {
+ yieldedRows++;
+ yield {
+ project_id: projectId,
+ trace_id: traceId,
+ span_id: `span-${yieldedRows}`,
+ parent_span_id: null,
+ start_time: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000000",
+ type: "GENERATION",
+ name: "generation",
+ input: "hello",
+ output: "世界",
+ metadata: { é: "界🙂" },
+ tool_definitions: {},
+ tool_calls: [],
+ tool_call_names: [],
+ };
+ }
+ },
+ );
+
+ for (let attempt = 0; attempt < 2; attempt++) {
+ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual(
+ {
+ observationCount: 4,
+ traceCount: 3,
+ projectCount: 2,
+ inputBytes: 20,
+ outputBytes: 24,
+ metadataBytes: 36,
+ ioMetadataBytes: 80,
+ },
+ );
+ }
+ expect(getTraceBatchEventStream).toHaveBeenCalledTimes(2);
+ expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload);
+ expect(yieldedRows).toBe(8);
+ for (const [name, bytes] of [
+ ["input_bytes", 20],
+ ["output_bytes", 24],
+ ["metadata_bytes", 36],
+ ["io_metadata_bytes", 80],
+ ] as const) {
+ expect(
+ vi
+ .mocked(recordDistribution)
+ .mock.calls.filter(
+ ([metric]) => metric === `langfuse.trace_batch.${name}`,
+ )
+ .map(([, value]) => value),
+ ).toEqual([bytes, bytes]);
+ }
+ expect(
+ vi
+ .mocked(recordDistribution)
+ .mock.calls.filter(
+ ([name]) => name === "langfuse.trace_batch.found_project_count",
+ )
+ .map(([, value]) => value),
+ ).toEqual([2, 2]);
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.missing_trace_count",
+ 2,
+ );
+ } finally {
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = ingestionEnabled;
+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = dispatcherEnabled;
+ }
+ });
+
+ it.each([true, false])(
+ "normalizes persisted single-project jobs and counts returned projects (has rows: %s)",
+ async (hasRows) => {
+ const trace = {
+ traceId: "trace",
+ minStart: 1_000,
+ maxStart: 2_000,
+ revision: "revision",
+ };
+ const job = {
+ data: {
+ id: "legacy-batch",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date().toISOString(),
+ payload: { projectId: "project", traces: [trace] },
+ },
+ } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+ vi.mocked(getTraceBatchEventStream).mockImplementation(
+ async function* () {
+ if (!hasRows) return;
+ yield {
+ project_id: "project",
+ trace_id: "trace",
+ span_id: "span",
+ parent_span_id: null,
+ start_time: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000000",
+ type: "GENERATION",
+ name: "generation",
+ input: "hello",
+ output: "world",
+ metadata: {},
+ tool_definitions: {},
+ tool_calls: [],
+ tool_call_names: [],
+ };
+ },
+ );
+
+ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({
+ observationCount: Number(hasRows),
+ traceCount: Number(hasRows),
+ projectCount: Number(hasRows),
+ inputBytes: hasRows ? 5 : 0,
+ outputBytes: hasRows ? 5 : 0,
+ metadataBytes: 0,
+ ioMetadataBytes: hasRows ? 10 : 0,
+ });
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.input_bytes",
+ hasRows ? 5 : 0,
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.output_bytes",
+ hasRows ? 5 : 0,
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.metadata_bytes",
+ 0,
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.found_project_count",
+ Number(hasRows),
+ );
+ expect(getTraceBatchEventStream).toHaveBeenCalledWith({
+ traces: [{ ...trace, projectId: "project" }],
+ });
+ },
+ );
+});
diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.ts
index ae71281f71..5d4377ef0a 100644
--- a/worker/src/queues/otelIngestionQueue.ts
+++ b/worker/src/queues/otelIngestionQueue.ts
@@ -36,6 +36,7 @@ import {
v4WritesToLegacyTables,
} from "../env";
import { IngestionService } from "../services/IngestionService";
+import { trackTraceBatchActivity } from "../features/traceBatching/traceBatching";
import { prisma } from "@langfuse/shared/src/db";
import { ClickhouseWriter } from "../services/ClickhouseWriter";
import {
@@ -830,6 +831,8 @@ export const otelIngestionQueueProcessorBuilder = (
? createObservationEvalSchedulerDeps()
: null;
+ const traceBatchEvents: { traceId: string; startTimeISO: string }[] = [];
+
await Promise.all(
// Process each event independently
eventInputs.map(async (eventInput) => {
@@ -880,6 +883,15 @@ export const otelIngestionQueueProcessorBuilder = (
if (shouldWriteToEventsTable) {
try {
await ingestionService.writeEventRecord(eventRecord);
+ if (
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED === "true"
+ ) {
+ traceBatchEvents.push({
+ traceId: eventInput.traceId,
+ startTimeISO: eventInput.startTimeISO,
+ });
+ }
} catch (error) {
traceException(error);
logger.error(
@@ -890,6 +902,15 @@ export const otelIngestionQueueProcessorBuilder = (
}
}),
);
+
+ if (traceBatchEvents.length > 0) {
+ recordDistribution(
+ "langfuse.trace_batch.ingestion_trace_count",
+ new Set(traceBatchEvents.map((event) => event.traceId)).size,
+ );
+ // The writer has accepted these records; its flush completes separately.
+ await trackTraceBatchActivity(projectId, traceBatchEvents);
+ }
} catch (e) {
const fileKey = job.data.payload.data.fileKey;
if (e instanceof ForbiddenError) {
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
new file mode 100644
index 0000000000..522b1c4278
--- /dev/null
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -0,0 +1,90 @@
+import { type Processor } from "bullmq";
+import {
+ getTraceBatchEventStream,
+ recordDistribution,
+ recordIncrement,
+ TraceBatchEventSchema,
+ type QueueName,
+ type TQueueJobTypes,
+} from "@langfuse/shared/src/server";
+import { env } from "../env";
+
+const JOB_MAX_AGE_MS = 2 * 60 * 60_000;
+
+export const traceBatchQueueProcessor: Processor<
+ TQueueJobTypes[QueueName.TraceBatch]
+> = async (job) => {
+ const disabledReason = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+ ? "not_cloud"
+ : env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true"
+ ? "reads_disabled"
+ : undefined;
+ if (disabledReason) {
+ // BullMQ reads these options after the processor returns, then removes atomically.
+ job.opts.removeOnComplete = true;
+ recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {
+ reason: disabledReason,
+ });
+ return { discarded: disabledReason };
+ }
+ const event = TraceBatchEventSchema.parse(job.data);
+ if (Date.now() - event.timestamp.getTime() >= JOB_MAX_AGE_MS) {
+ job.opts.removeOnComplete = true;
+ recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {
+ reason: "expired",
+ });
+ return { discarded: "expired" };
+ }
+ const batch = event.payload;
+ const foundTraces = new Set<string>();
+ const foundProjects = new Set<string>();
+ let observationCount = 0;
+ let inputBytes = 0;
+ let outputBytes = 0;
+ let metadataBytes = 0;
+
+ for await (const event of getTraceBatchEventStream(batch)) {
+ observationCount++;
+ foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));
+ foundProjects.add(event.project_id);
+ // Logical UTF-8 payload size, excluding JSON transport and compression.
+ inputBytes += Buffer.byteLength(event.input);
+ outputBytes += Buffer.byteLength(event.output);
+ for (const [key, value] of Object.entries(event.metadata)) {
+ metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);
+ }
+ }
+
+ recordDistribution(
+ "langfuse.trace_batch.observation_count",
+ observationCount,
+ );
+ recordDistribution(
+ "langfuse.trace_batch.found_trace_count",
+ foundTraces.size,
+ );
+ const ioMetadataBytes = inputBytes + outputBytes + metadataBytes;
+ recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);
+ recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);
+ recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);
+ recordDistribution("langfuse.trace_batch.io_metadata_bytes", ioMetadataBytes);
+ recordDistribution(
+ "langfuse.trace_batch.found_project_count",
+ foundProjects.size,
+ );
+ recordDistribution(
+ "langfuse.trace_batch.missing_trace_count",
+ batch.traces.length - foundTraces.size,
+ );
+
+ // Retries can repeat this read; observation payloads never enter job results.
+ return {
+ observationCount,
+ traceCount: foundTraces.size,
+ projectCount: foundProjects.size,
+ inputBytes,
+ outputBytes,
+ metadataBytes,
+ ioMetadataBytes,
+ };
+};
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index 3717e49d6f..812fe1ce59 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -24,6 +24,7 @@ import {
queueMetricsRunner,
monitorRunners,
inAppAgentDlqRetryRunner,
+ traceBatchDispatcher,
} from "../app";
export const onShutdown: NodeJS.SignalsListener = async (signal) => {
@@ -50,6 +51,9 @@ const runDrainAndClose = async () => {
server?.close();
logger.info("Server has been closed.");
+ // Give in-flight dispatch up to five seconds before continuing shutdown.
+ await traceBatchDispatcher?.drain();
+
// Stop batch project cleaners
for (const cleaner of batchProjectCleaners) {
cleaner.stop();