🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): Add cloud-only trace batch PoC (baseline) (#17495)
Lysander Kiesel0e62b3f68bf5
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex 55a6c4ccc5..37bd3f3d90 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -23,6 +23,10 @@- Repository layer: `src/server/repositories/*`- Queue payload schemas: `src/server/queues.ts`- Queue helpers: `src/server/redis/*`+- Internal trace-batch queue: `src/server/redis/traceBatch.ts` (cloud-gated);+ payloads in `src/server/queues.ts` accept persisted single-project jobs.+ Full-event streaming reads live in `src/server/repositories/trace-batch.ts`;+ the worker owns experiment enablement and lifecycle.- Code evaluator dispatcher/error contract: `src/server/evals/codeEvalDispatcherTypes.ts`. Keep provider mappings, user-visible messages, and worker terminal-outcome classification aligned when adding an error code.- Dashboard/monitor query feature (data model + server-only builder/executor): `src/features/query/*`- Query-builder AST (server half, WIP): `src/server/query-ast/*` — golden-SQLdiff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex ce1e6a7796..adc85bfb14 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -101,6 +101,7 @@ export * from "./redis/webhookQueue";export * from "./redis/monitorQueue";export * from "./redis/inAppAgentRunQueue";export * from "./redis/traceDelete";+export * from "./redis/traceBatch";export * from "./redis/projectDelete";export * from "./redis/scoreDelete";export * from "./redis/datasetDelete";diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex eadebcf135..afaaccb62d 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -595,7 +595,7 @@ const AGGREGATION_FIELD_SETS = {* // Use when you need to query across all projects (use with caution!)* const builder = new EventsQueryBuilder({ projectId: NoProjectId });*/-const NoProjectId = Symbol("NoProjectId");+export const NoProjectId = Symbol("NoProjectId");export type NoProjectIdType = typeof NoProjectId;/**diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.tsindex 7d995ea025..531964244f 100644--- a/packages/shared/src/server/queues.ts+++ b/packages/shared/src/server/queues.ts@@ -404,6 +404,7 @@ export const RetryBaggage = z.object({export type RetryBaggage = z.infer<typeof RetryBaggage>;export enum QueueName {+ TraceBatch = "trace-batch",TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsertTraceDelete = "trace-delete",ProjectDelete = "project-delete",@@ -446,6 +447,7 @@ export enum QueueName {}export enum QueueJobs {+ TraceBatch = "trace-batch",TraceUpsert = "trace-upsert",TraceDelete = "trace-delete",ProjectDelete = "project-delete",@@ -484,7 +486,37 @@ export enum QueueJobs {V4LegacyApiUsageJob = "v4-legacy-api-usage-job",}+export const TraceBatchTraceSchema = z.object({+ projectId: z.string(),+ traceId: z.string(),+ minStart: z.number(),+ maxStart: z.number(),+ revision: z.string(),+});++export const TraceBatchEventSchema = z.object({+ timestamp: z.coerce.date(),+ id: z.string(),+ name: z.literal(QueueJobs.TraceBatch),+ payload: z.union([+ z.object({ traces: z.array(TraceBatchTraceSchema).min(1) }).strict(),+ // Persisted single-project jobs must remain readable while consumers drain.+ z+ .object({+ projectId: z.string(),+ traces: z+ .array(TraceBatchTraceSchema.omit({ projectId: true }).strict())+ .min(1),+ })+ .strict()+ .transform(({ projectId, traces }) => ({+ traces: traces.map((trace) => ({ ...trace, projectId })),+ })),+ ]),+});+export type TQueueJobTypes = {+ [QueueName.TraceBatch]: z.infer<typeof TraceBatchEventSchema>;[QueueName.TraceUpsert]: {timestamp: Date;id: string;diff --git a/packages/shared/src/server/redis/getQueue.ts b/packages/shared/src/server/redis/getQueue.tsindex 4e65c0bfa0..62dc54f851 100644--- a/packages/shared/src/server/redis/getQueue.ts+++ b/packages/shared/src/server/redis/getQueue.ts@@ -7,6 +7,7 @@ import { CloudFreeTierUsageThresholdQueue } from "./cloudFreeTierUsageThresholdQimport { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";import { ExperimentCreateQueue } from "./experimentCreateQueue";import { TraceDeleteQueue } from "./traceDelete";+import { TraceBatchQueue } from "./traceBatch";import { ProjectDeleteQueue } from "./projectDelete";import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";import { PostHogIntegrationProcessingQueue } from "./postHogIntegrationProcessingQueue";@@ -48,6 +49,8 @@ export function getQueue(>,): Queue | null {switch (queueName) {+ case QueueName.TraceBatch:+ return TraceBatchQueue.getInstance();case QueueName.BatchExport:return BatchExportQueue.getInstance();case QueueName.CloudUsageMeteringQueue:diff --git a/packages/shared/src/server/redis/traceBatch.test.ts b/packages/shared/src/server/redis/traceBatch.test.tsnew file mode 100644index 0000000000..77553739fa--- /dev/null+++ b/packages/shared/src/server/redis/traceBatch.test.ts@@ -0,0 +1,45 @@+import { expect, it, vi } from "vitest";++const { env, createQueueOptions, queueCreated } = vi.hoisted(() => ({+ env: {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined as string | undefined,+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: "true",+ LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED: "true",+ QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED: "true",+ LANGFUSE_TRACE_BATCH_READ_ENABLED: "true",+ },+ createQueueOptions: vi.fn(() => ({ connection: {} })),+ queueCreated: vi.fn(),+}));++vi.mock("../../env", () => ({ env }));+vi.mock("./redis", () => ({+ createBullMQQueueOptionsWithRedis: createQueueOptions,+}));+vi.mock("../logger", () => ({ logger: { error: vi.fn() } }));+vi.mock("bullmq", () => ({+ Queue: class {+ constructor() {+ queueCreated();+ }+ on = vi.fn();+ },+}));++import { TraceBatchQueue } from "./traceBatch";++it("keeps queue inspection inert outside cloud even with all experiment flags enabled", () => {+ expect(TraceBatchQueue.getInstance()).toBeNull();+ expect(createQueueOptions).not.toHaveBeenCalled();+ expect(queueCreated).not.toHaveBeenCalled();++ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+ const cloudQueue = TraceBatchQueue.getInstance();+ expect(cloudQueue).not.toBeNull();+ expect(TraceBatchQueue.getInstance()).toBe(cloudQueue);+ expect(queueCreated).toHaveBeenCalledTimes(1);++ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;+ expect(TraceBatchQueue.getInstance()).toBeNull();+ expect(queueCreated).toHaveBeenCalledTimes(1);+});diff --git a/packages/shared/src/server/redis/traceBatch.ts b/packages/shared/src/server/redis/traceBatch.tsnew file mode 100644index 0000000000..3c994ab9b4--- /dev/null+++ b/packages/shared/src/server/redis/traceBatch.ts@@ -0,0 +1,35 @@+import { Queue } from "bullmq";+import { env } from "../../env";+import { QueueName, type TQueueJobTypes } from "../queues";+import { createBullMQQueueOptionsWithRedis } from "./redis";+import { logger } from "../logger";++export class TraceBatchQueue {+ private static instance: Queue<TQueueJobTypes[QueueName.TraceBatch]> | null =+ null;++ public static getInstance() {+ if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) return null;+ if (this.instance) return this.instance;+ const options = createBullMQQueueOptionsWithRedis(QueueName.TraceBatch);+ if (!options) return null;++ this.instance = new Queue<TQueueJobTypes[QueueName.TraceBatch]>(+ QueueName.TraceBatch,+ {+ ...options,+ defaultJobOptions: {+ // Retain recent job IDs so a dispatcher retry can reuse its delivery.+ removeOnComplete: { age: 3600, count: 10000 },+ removeOnFail: 1000,+ attempts: 3,+ backoff: { type: "exponential", delay: 5000 },+ },+ },+ );+ this.instance.on("error", (error) => {+ logger.error("TraceBatchQueue error", error);+ });+ return this.instance;+ }+}diff --git a/packages/shared/src/server/repositories/index.ts b/packages/shared/src/server/repositories/index.tsindex c4e2d6924e..f55c331ee7 100644--- a/packages/shared/src/server/repositories/index.ts+++ b/packages/shared/src/server/repositories/index.ts@@ -3,6 +3,7 @@ export * from "./traces";export * from "./observations";export * from "./events";export * from "./events-stream";+export * from "./trace-batch";export * from "./types";export * from "./dashboards";export * from "./traces_converters";diff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.tsnew file mode 100644index 0000000000..74fbf53e74--- /dev/null+++ b/packages/shared/src/server/repositories/trace-batch.ts@@ -0,0 +1,108 @@+import {+ EventsQueryBuilder,+ NoProjectId,+} from "../queries/clickhouse-sql/event-query-builder";+import { queryClickhouseStream, TupleParam } from "./clickhouse";++type TraceBatchEventRow = {+ project_id: string;+ trace_id: string;+ span_id: string;+ parent_span_id: string | null;+ start_time: string;+ event_ts: string;+ type: string;+ name: string;+ input: string;+ output: string;+ metadata: Record<string, string>;+ tool_definitions: Record<string, string>;+ tool_calls: string[];+ tool_call_names: string[];+};++const TRACE_QUERY_BUFFER_MS = 2 * 60_000;++/**+ * Stream full events_full payloads into the worker without retaining a whole batch.+ * Rows reflect the events table's current merge state, as in other event reads;+ * this read-only experiment does not deduplicate versions or enrich model data.+ */+export async function* getTraceBatchEventStream(props: {+ traces: ReadonlyArray<{+ projectId: string;+ traceId: string;+ minStart: number;+ maxStart: number;+ }>;+}): AsyncGenerator<TraceBatchEventRow> {+ if (props.traces.length === 0) return;++ const projectIds = [...new Set(props.traces.map((trace) => trace.projectId))];+ const traceIds = [...new Set(props.traces.map((trace) => trace.traceId))];+ const builder = new EventsQueryBuilder({ projectId: NoProjectId })+ .selectRaw(+ "e.project_id",+ "e.trace_id",+ "e.span_id",+ "e.parent_span_id",+ "e.start_time",+ "e.event_ts",+ "e.type",+ "e.name",+ )+ // Load full input/output (false = no truncation) and every metadata key.+ .selectIO(false)+ .selectMetadataExpanded()+ .selectFieldSet("tools")+ // Read events_full rather than the smaller, truncated events_core table.+ .forceFullTable()+ // Separate filters retain project-prefix and trace-index pruning.+ .whereRaw("e.project_id IN ({projectIds: Array(String)})", { projectIds })+ .whereRaw("e.trace_id IN ({traceIds: Array(String)})", { traceIds })+ // Independent IN lists also match crossed pairs. Only these exact tenant/+ // trace pairs may return payloads, even when projects share trace IDs.+ .whereRaw(+ "(e.project_id, e.trace_id) IN {tracePairs: Array(Tuple(String, String))}",+ {+ tracePairs: props.traces.map(+ ({ projectId, traceId }) => new TupleParam([projectId, traceId]),+ ),+ },+ )+ // Equality filters add this primary-key hash condition automatically; IN+ // needs it explicitly. The exact IDs above also exclude hash collisions.+ .whereRaw(+ "xxHash32(e.trace_id) IN (SELECT arrayJoin(arrayMap(id -> xxHash32(id), {traceIds: Array(String)})))",+ )+ // One shared window enables partition/granule pruning. It can include+ // observations beyond a selected trace's own bounds when another trace+ // widens the batch window; per-trace predicates can narrow that coverage.+ .whereRaw(+ "e.start_time >= fromUnixTimestamp64Milli({batchMinStart: Int64}) AND e.start_time <= fromUnixTimestamp64Milli({batchMaxStart: Int64})",+ {+ batchMinStart:+ Math.min(...props.traces.map((trace) => trace.minStart)) -+ TRACE_QUERY_BUFFER_MS,+ batchMaxStart:+ Math.max(...props.traces.map((trace) => trace.maxStart)) ++ TRACE_QUERY_BUFFER_MS,+ },+ );++ const { query, params } = builder.buildWithParams();+ yield* queryClickhouseStream<TraceBatchEventRow>({+ query,+ params,+ tags: {+ projectId: projectIds.length === 1 ? projectIds[0] : "MULTI_PROJECT",+ },+ preferredClickhouseService: "EventsReadOnly",+ // Bound background-read CPU/time; timeouts fail instead of returning partial results.+ clickhouseSettings: {+ max_threads: 2,+ max_execution_time: 30,+ timeout_overflow_mode: "throw",+ },+ });+}diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex 79a1dc0775..564c57d035 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -1,6 +1,7 @@import {createEvent,createEventsCh,+ getTraceBatchEventStream,getObservationsForTraceFromEventsTable,getObservationsWithModelDataFromEventsTable,getObservationsCountFromEventsTable,@@ -64,6 +65,178 @@ const findFilterOption = () => rows.find((row) => row.column === column && row.value === value);describe("Clickhouse Events Repository Test", () => {+ it("streams complete trace batches with full payloads, exact tenant/trace pairs and shared batch time bounds", async () => {+ const batchProjectId = randomUUID();+ const otherProjectId = randomUUID();+ const firstTraceId = randomUUID();+ const secondTraceId = randomUUID();+ const otherTraceId = randomUUID();+ const start = Date.now();+ const buffer = 2 * 60_000;+ const input = JSON.stringify({ message: "input".repeat(100) });+ const output = JSON.stringify({ message: "output".repeat(100) });+ const metadataValue = JSON.stringify({ detail: "metadata".repeat(100) });+ const first = createEvent({+ project_id: batchProjectId,+ trace_id: firstTraceId,+ start_time: new Date(start - buffer),+ input,+ output,+ metadata_names: ["context"],+ metadata_values: [metadataValue],+ tool_definitions: { search: '{"description":"Search"}' },+ tool_calls: ['{"name":"search"}'],+ tool_call_names: ["search"],+ });+ const second = createEvent({+ project_id: batchProjectId,+ trace_id: secondTraceId,+ start_time: new Date(start + 600_000 + buffer),+ });+ // Another trace can widen the shared window beyond this trace's own bounds.+ const insideBatchWindow = createEvent({+ project_id: batchProjectId,+ trace_id: firstTraceId,+ start_time: new Date(start + buffer + 1),+ });+ await createEventsCh([+ first,+ second,+ insideBatchWindow,+ createEvent({+ project_id: otherProjectId,+ trace_id: otherTraceId,+ start_time: new Date(start),+ }),+ // The same trace ID in two projects represents two different traces.+ createEvent({+ project_id: otherProjectId,+ trace_id: firstTraceId,+ start_time: new Date(start),+ }),+ // These crossed pairs match both independent IN lists, but are unrequested.+ createEvent({+ project_id: batchProjectId,+ trace_id: otherTraceId,+ start_time: new Date(start),+ }),+ createEvent({+ project_id: otherProjectId,+ trace_id: secondTraceId,+ start_time: new Date(start),+ }),+ createEvent({+ project_id: randomUUID(),+ trace_id: firstTraceId,+ start_time: new Date(start),+ }),+ createEvent({+ project_id: batchProjectId,+ trace_id: randomUUID(),+ start_time: new Date(start),+ }),+ createEvent({+ project_id: batchProjectId,+ trace_id: firstTraceId,+ start_time: new Date(start - buffer - 1),+ }),+ createEvent({+ project_id: batchProjectId,+ trace_id: secondTraceId,+ start_time: new Date(start + 600_000 + buffer + 1),+ }),+ ]);++ // Exceed the single-trace reader's historical 20,000-observation cap.+ const extraRowCount = 20_001;+ const template = createEvent({+ project_id: batchProjectId,+ trace_id: firstTraceId,+ start_time: new Date(start),+ input: "",+ output: "",+ metadata_names: [],+ metadata_values: [],+ });+ for (let offset = 0; offset < extraRowCount; offset += 2_000) {+ await createEventsCh(+ Array.from({ length: Math.min(2_000, extraRowCount - offset) }, () => {+ const spanId = randomUUID();+ return { ...template, span_id: spanId, id: spanId };+ }),+ );+ }++ let rowCount = 0;+ let fullPayloadSeen = false;+ let insideBatchWindowSeen = false;+ const foundTraces = new Set<string>();+ for await (const event of getTraceBatchEventStream({+ traces: [+ {+ projectId: batchProjectId,+ traceId: firstTraceId,+ minStart: start,+ maxStart: start,+ },+ {+ projectId: batchProjectId,+ traceId: secondTraceId,+ minStart: start + 600_000,+ maxStart: start + 600_000,+ },+ {+ projectId: otherProjectId,+ traceId: otherTraceId,+ minStart: start,+ maxStart: start,+ },+ {+ projectId: otherProjectId,+ traceId: firstTraceId,+ minStart: start,+ maxStart: start,+ },+ // Exercise the maximum configurable batch size. Missing IDs stress+ // HTTP parameters and query size without unrelated fixture rows.+ ...Array.from({ length: 996 }, () => ({+ projectId: batchProjectId,+ traceId: randomUUID(),+ minStart: start,+ maxStart: start,+ })),+ ],+ })) {+ rowCount++;+ foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));+ if (event.span_id === insideBatchWindow.span_id) {+ insideBatchWindowSeen = true;+ }+ if (event.span_id === first.span_id) {+ fullPayloadSeen = true;+ expect(event).toMatchObject({+ input,+ output,+ metadata: { context: metadataValue },+ tool_definitions: first.tool_definitions,+ tool_calls: first.tool_calls,+ tool_call_names: first.tool_call_names,+ });+ }+ }+ expect(fullPayloadSeen).toBe(true);+ expect(insideBatchWindowSeen).toBe(true);+ expect(foundTraces).toEqual(+ new Set([+ JSON.stringify([batchProjectId, firstTraceId]),+ JSON.stringify([batchProjectId, secondTraceId]),+ JSON.stringify([otherProjectId, otherTraceId]),+ JSON.stringify([otherProjectId, firstTraceId]),+ ]),+ );+ expect(rowCount).toBe(extraRowCount + 5);+ }, 60_000);+it("should kill redis connection", () => {// we need at least one test case to avoid hanging// redis connection when everything else is skipped.diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex cba0983dd7..5fdbfd51a7 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -16,6 +16,11 @@- Worker registration/lifecycle: `src/queues/workerManager.ts`- Queue processors: `src/queues/*`- Feature processors: `src/features/*`+- Internal cloud trace batching: `src/features/traceBatching/traceBatching.ts` and+ `src/queues/traceBatchQueue.ts`; controls and Redis lifecycle are documented in+ `src/features/traceBatching/README.md`. Keep producer, dispatcher, consumer and reads+ independently default-off and cloud-gated. Do not expose these PoC controls in+ local or production env templates.- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.- Service layer: `src/services/*`- Rust addon (`@langfuse/native`): telemetry init and the startup hello call livediff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsnew file mode 100644index 0000000000..e52ccf7f4c--- /dev/null+++ b/worker/src/__tests__/traceBatching.test.ts@@ -0,0 +1,954 @@+import { randomBytes, randomUUID } from "node:crypto";+import { Queue, QueueEvents, Worker } from "bullmq";+import {+ afterAll,+ afterEach,+ beforeEach,+ describe,+ expect,+ it,+ vi,+} from "vitest";+import {+ createNewRedisInstance,+ getQueuePrefix,+ getS3EventStorageClient,+ getTraceBatchEventStream,+ QueueJobs,+ QueueName,+ recordDistribution,+ recordIncrement,+ redis,+ TraceBatchEventSchema,+ TraceBatchQueue,+ type TQueueJobTypes,+} from "@langfuse/shared/src/server";+import * as shared from "@langfuse/shared/src/server";+import { env } from "../env";+import {+ trackTraceBatchActivity,+ TraceBatchDispatcher,+} from "../features/traceBatching/traceBatching";+import { otelIngestionQueueProcessorBuilder } from "../queues/otelIngestionQueue";+import { traceBatchQueueProcessor } from "../queues/traceBatchQueue";+import { ClickhouseWriter } from "../services/ClickhouseWriter";++vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const original =+ await importOriginal<typeof import("@langfuse/shared/src/server")>();+ return {+ ...original,+ redis: original.createNewRedisInstance({+ keyPrefix: `trace-batch-test-${randomUUID()}:`,+ }),+ recordDistribution: vi.fn(),+ recordIncrement: vi.fn(),+ getS3EventStorageClient: vi.fn(),+ };+});++vi.mock("../features/evaluation/observationEval", async (importOriginal) => ({+ ...(await importOriginal<+ typeof import("../features/evaluation/observationEval")+ >()),+ fetchObservationEvalRules: vi.fn().mockResolvedValue([]),+}));++describe("trace micro-batch scheduling with Redis", () => {+ const dueKey = "{trace-batch}:due";+ const stateKey = "{trace-batch}:state";+ const originalEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;+ const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;+ const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;+ const originalSamplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;+ const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;+ const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;+ const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;+ let queue: Queue<TQueueJobTypes[QueueName.TraceBatch]>;+ let connection: NonNullable<ReturnType<typeof createNewRedisInstance>>;+ const runners: TraceBatchDispatcher[] = [];+ const client = () => {+ if (!redis) throw new Error("Redis is required for this test");+ return redis;+ };+ const member = (projectId: string, traceId: string) =>+ JSON.stringify([projectId, traceId]);+ const event = (traceId: string, start = 1_000_000) => ({+ traceId,+ startTimeISO: new Date(start).toISOString(),+ });+ const runner = () => {+ const instance = new TraceBatchDispatcher();+ runners.push(instance);+ return instance;+ };+ async function makeDue(projectId: string, ...traceIds: string[]) {+ for (const traceId of traceIds)+ await client().zadd(+ dueKey,+ Date.now() - 10_000,+ member(projectId, traceId),+ );+ }++ beforeEach(async () => {+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "true";+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 60;+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 7_200_000;+ await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");+ const redisConnection = createNewRedisInstance();+ if (!redisConnection) throw new Error("Redis is required for this test");+ connection = redisConnection;+ const name = `trace-batch-test-${randomUUID()}`;+ queue = new Queue(name, { connection, prefix: getQueuePrefix(name) });+ await queue.waitUntilReady();+ vi.spyOn(TraceBatchQueue, "getInstance").mockReturnValue(queue);+ });++ afterEach(async () => {+ for (const dispatcher of runners) await dispatcher.drain();+ runners.length = 0;+ vi.restoreAllMocks();+ vi.clearAllMocks();+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = originalEnabled;+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = originalSamplingRate;+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;+ await queue.obliterate({ force: true });+ await queue.close();+ connection.disconnect();+ await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");+ });+ afterAll(() => client().disconnect());++ it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;+ const evaluate = vi.spyOn(client(), "eval");+ const lock = vi.spyOn(client(), "set");+ await trackTraceBatchActivity("project", [event("trace")]);+ await runner()["execute"]();+ expect(evaluate).not.toHaveBeenCalled();+ expect(lock).not.toHaveBeenCalled();+ expect(await client().exists(dueKey, stateKey)).toBe(0);+ expect(await queue.count()).toBe(0);+ });++ it("ingests through BullMQ, postpones readiness on new activity, and consumes full ClickHouse payloads once due", async () => {+ const original = {+ LANGFUSE_TRACE_BATCH_IDLE_MS: env.LANGFUSE_TRACE_BATCH_IDLE_MS,+ LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,+ LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED:+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,+ LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED:+ env.LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED,+ };+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 4_000;+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0.1;+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED = "false";+ env.LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED = "false";++ const ingestionName = `trace-batch-ingestion-test-${randomUUID()}`;+ const ingestionQueue = new Queue<+ TQueueJobTypes[QueueName.OtelIngestionQueue]+ >(ingestionName, { connection, prefix: getQueuePrefix(ingestionName) });+ const ingestionEvents = new QueueEvents(ingestionName, {+ connection,+ prefix: ingestionQueue.opts.prefix,+ });+ const ingestionWorker = new Worker(+ ingestionName,+ otelIngestionQueueProcessorBuilder(false),+ { connection, prefix: ingestionQueue.opts.prefix },+ );+ const batchEvents = new QueueEvents(queue.name, {+ connection,+ prefix: queue.opts.prefix,+ });+ const batchWorker = new Worker(queue.name, traceBatchQueueProcessor, {+ connection,+ prefix: queue.opts.prefix,+ });+ const writer = ClickhouseWriter.getInstance();++ try {+ await Promise.all([+ ingestionWorker.waitUntilReady(),+ ingestionEvents.waitUntilReady(),+ batchWorker.waitUntilReady(),+ batchEvents.waitUntilReady(),+ ]);+ const projectId = randomUUID();+ // The evaluator sampler assigns these scores 0.0593, 0.0291 and 0.7116.+ const traceId = "00000000000000000000000000000001";+ const otherTraceId = "00000000000000000000000000000003";+ const excludedTraceId = "00000000000000000000000000000002";+ const input = JSON.stringify({ message: "input".repeat(1_000) });+ const output = JSON.stringify({ message: "output".repeat(1_000) });+ const metadata = JSON.stringify({ context: "metadata".repeat(1_000) });+ const nano = BigInt(Date.now()) * 1_000_000n;+ const timestamp = {+ low: Number(nano & 0xffffffffn),+ high: Number(nano >> 32n),+ unsigned: true,+ };+ async function ingest(traceIds: string[]) {+ // S3 transport and unrelated evaluator configuration are stubbed;+ // OTLP conversion, writer, Redis scripts and both queue workers are real.+ vi.mocked(getS3EventStorageClient).mockReturnValue({+ download: vi.fn().mockResolvedValue(+ JSON.stringify([+ {+ resource: { attributes: [] },+ scopeSpans: [+ {+ scope: { name: "langfuse-sdk", version: "4.0.0" },+ spans: traceIds.map((id) => ({+ traceId: Buffer.from(id, "hex").toJSON(),+ spanId: randomBytes(8).toJSON(),+ name: "trace-batch-integration",+ kind: 1,+ startTimeUnixNano: timestamp,+ endTimeUnixNano: timestamp,+ attributes: Object.entries({+ "langfuse.observation.type": "span",+ "langfuse.observation.input": input,+ "langfuse.observation.output": output,+ "langfuse.observation.metadata": metadata,+ }).map(([key, value]) => ({+ key,+ value: { stringValue: value },+ })),+ })),+ },+ ],+ },+ ]),+ ),+ } as unknown as ReturnType<typeof getS3EventStorageClient>);+ const job = await ingestionQueue.add(QueueJobs.OtelIngestionJob, {+ id: randomUUID(),+ timestamp: new Date(),+ name: QueueJobs.OtelIngestionJob,+ payload: {+ data: { fileKey: "trace-batch-integration.json" },+ authCheck: {+ validKey: true,+ scope: { projectId, orgId: randomUUID(), accessLevel: "project" },+ },+ ingestionVersion: "4",+ },+ });+ await job.waitUntilFinished(ingestionEvents, 10_000);+ }+ async function waitUntilRedisTime(deadline: number) {+ await expect+ .poll(+ async () => {+ const [seconds, microseconds] = await client().time();+ return (+ Number(seconds) * 1_000 ++ Math.floor(Number(microseconds) / 1_000)+ );+ },+ { interval: 25, timeout: 10_000 },+ )+ .toBeGreaterThanOrEqual(deadline);+ }++ await ingest([traceId, excludedTraceId]);+ expect(await client().hlen(stateKey)).toBe(1);+ const firstDue = Number(+ await client().zscore(dueKey, member(projectId, traceId)),+ );+ await waitUntilRedisTime(firstDue - 2_000);+ // Leave ample time to check the old deadline even on a busy CI worker.+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;+ await ingest([traceId, otherTraceId, excludedTraceId]);+ const renewedDue = Number(+ await client().zscore(dueKey, member(projectId, traceId)),+ );+ expect(renewedDue).toBeGreaterThan(firstDue);+ expect(await client().hlen(stateKey)).toBe(2);++ const dispatcher = runner();+ await waitUntilRedisTime(firstDue);+ await dispatcher.processBatch();+ expect(await queue.getJobCounts("wait", "active", "completed")).toEqual({+ wait: 0,+ active: 0,+ completed: 0,+ });++ await writer.flushAll(true);+ // Sampling only gates readiness; excluded observations are still ingested.+ let excludedObservations = 0;+ for await (const observation of getTraceBatchEventStream({+ traces: [+ {+ projectId,+ traceId: excludedTraceId,+ minStart: Number(nano / 1_000_000n),+ maxStart: Number(nano / 1_000_000n),+ },+ ],+ })) {+ expect(observation.trace_id).toBe(excludedTraceId);+ excludedObservations++;+ }+ expect(excludedObservations).toBe(2);+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0;+ await waitUntilRedisTime(renewedDue);+ await dispatcher.processBatch();+ const [batch] = await queue.getJobs(["wait", "active", "completed"]);+ expect(+ batch.data.payload.traces.map((trace) => trace.traceId).sort(),+ ).toEqual([traceId, otherTraceId].sort());+ const result = await batch.waitUntilFinished(batchEvents, 10_000);+ expect(result).toMatchObject({+ observationCount: 3,+ traceCount: 2,+ projectCount: 1,+ });+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.found_project_count",+ 1,+ );+ // All three observations must carry their untruncated I/O and metadata.+ expect(result.ioMetadataBytes).toBeGreaterThanOrEqual(+ 3 * (Buffer.byteLength(input) + Buffer.byteLength(output) + 8_000),+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.size",+ 2,+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.io_metadata_bytes",+ result.ioMetadataBytes,+ );+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ await dispatcher.processBatch();+ expect(await queue.getCompletedCount()).toBe(1);+ } finally {+ try {+ await Promise.all([ingestionWorker.close(), batchWorker.close()]);+ await Promise.all([ingestionEvents.close(), batchEvents.close()]);+ await ingestionQueue.obliterate({ force: true });+ await ingestionQueue.close();+ await writer.shutdown();+ } finally {+ Object.assign(env, original);+ }+ }+ }, 30_000);++ it("keeps stable nested trace samples across replay and counts decisions once per trace in each ingestion batch", async () => {+ const traceIds = Array.from({ length: 1_000 }, (_, i) =>+ i.toString(16).padStart(32, "0"),+ );+ let previous: string[] = [];+ // Fixed fixtures pin the evaluator's sampling cohort, including both endpoints.+ for (const [rate, expectedCount] of [+ [0, 0],+ [0.1, 78],+ [0.1, 78],+ [0.5, 478],+ [1, 1_000],+ ]) {+ await client().del(dueKey, stateKey);+ vi.mocked(recordIncrement).mockClear();+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = rate;+ traceIds.reverse();+ await trackTraceBatchActivity(+ "project",+ traceIds.flatMap((id) => [+ event(id, 100),+ event(id, 200),+ event(id, 150),+ ]),+ );+ const selected = (await client().hkeys(stateKey)).sort();+ expect(selected).toHaveLength(expectedCount);+ expect(await client().zcard(dueKey)).toBe(expectedCount);+ expect(selected).toEqual(expect.arrayContaining(previous));+ if (previous.length === expectedCount) expect(selected).toEqual(previous);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.sampling_decisions",+ expectedCount,+ { decision: "selected" },+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.sampling_decisions",+ 1_000 - expectedCount,+ { decision: "excluded" },+ );+ previous = selected;+ }+ });++ it("preserves min/max across concurrent out-of-order arrivals, resets readiness, and separates tenants", async () => {+ await Promise.all(+ [30, 0, 20, 10].map((start) =>+ trackTraceBatchActivity("project", [event("trace", start)]),+ ),+ );+ const state = JSON.parse(+ (await client().hget(stateKey, member("project", "trace")))!,+ );+ expect(state).toMatchObject({ minStart: 0, maxStart: 30 });+ await makeDue("project", "trace");+ await trackTraceBatchActivity("project", [event("trace", 15)]);+ expect(+ Number(await client().zscore(dueKey, member("project", "trace"))),+ ).toBeGreaterThan(Date.now() + 590_000);+ await trackTraceBatchActivity("other", [event("trace", 100)]);+ expect(+ JSON.parse((await client().hget(stateKey, member("other", "trace")))!),+ ).toMatchObject({ minStart: 100, maxStart: 100 });+ await runner().processBatch();+ expect(await queue.getWaitingCount()).toBe(0);+ });++ it.each([+ { maxSize: 60, sizes: [22, 60, 60, 60], projectCounts: [1, 1, 1, 2] },+ { maxSize: 80, sizes: [42, 80, 80], projectCounts: [1, 1, 2] },+ ])(+ "dispatches cross-project batches capped at $maxSize, records their distribution, and drains with intake off",+ async ({ maxSize, sizes, projectCounts }) => {+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = maxSize;+ const traces = Array.from(+ { length: 201 },+ (_, index) => `trace-${index}`,+ );+ await trackTraceBatchActivity(+ "project",+ traces.map((traceId) => event(traceId)),+ );+ await trackTraceBatchActivity("other", [event("trace-0")]);+ await client().zadd(+ dueKey,+ ...traces.flatMap((traceId, index) => [+ Date.now() - 1_000 + index,+ member("project", traceId),+ ]),+ );+ await makeDue("other", "trace-0");+ const add = vi.spyOn(queue, "add");+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";+ await trackTraceBatchActivity("disabled", [event("ignored")]);+ await runner().processBatch();+ const jobs = await queue.getJobs(["wait"]);+ for (const job of jobs) TraceBatchEventSchema.parse(job.data);+ expect(+ jobs.map((job) => job.data.payload.traces.length).sort((a, b) => a - b),+ ).toEqual(sizes);+ expect(+ jobs+ .flatMap((job) =>+ job.data.payload.traces.map((trace) =>+ member(trace.projectId, trace.traceId),+ ),+ )+ .sort(),+ ).toEqual(+ [+ ...traces.map((traceId) => member("project", traceId)),+ member("other", "trace-0"),+ ].sort(),+ );+ expect(+ add.mock.calls.flatMap(([, job]) =>+ job.payload.traces.map((trace) =>+ member(trace.projectId, trace.traceId),+ ),+ ),+ ).toEqual([+ member("other", "trace-0"),+ ...traces.map((traceId) => member("project", traceId)),+ ]);+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ expect(+ vi+ .mocked(recordDistribution)+ .mock.calls.filter(([name]) => name === "langfuse.trace_batch.size")+ .map(([, value]) => value)+ .sort((a, b) => Number(a) - Number(b)),+ ).toEqual(sizes);+ expect(+ vi+ .mocked(recordDistribution)+ .mock.calls.filter(+ ([name]) => name === "langfuse.trace_batch.project_count",+ )+ .map(([, value]) => value)+ .sort((a, b) => Number(a) - Number(b)),+ ).toEqual(projectCounts);+ await runner().processBatch();+ expect(await queue.getWaitingCount()).toBe(sizes.length);+ },+ );++ it("groups the complete due cohort by sorted project across hydration chunks and the former run limit", async () => {+ const due = Date.now() - 100_000;+ const traces = Array.from({ length: 10_061 }, (_, i) => ({+ projectId: ["project-c", "project-a", "project-b"][i % 3],+ traceId: `trace-${String(i).padStart(5, "0")}`,+ due: due + Math.floor(i / 2_000) * 1_000,+ }));+ for (const projectId of ["project-c", "project-a", "project-b"]) {+ await trackTraceBatchActivity(+ projectId,+ traces+ .filter((trace) => trace.projectId === projectId)+ .map((trace) => event(trace.traceId)),+ );+ }+ await client().zadd(+ dueKey,+ ...traces.flatMap((trace) => [+ trace.due,+ member(trace.projectId, trace.traceId),+ ]),+ );+ const add = vi.spyOn(queue, "add");+ await runner().processBatch();+ const batches = add.mock.calls.map(([, job]) => job.payload.traces);+ expect(batches.map((batch) => batch.length)).toEqual([+ ...Array(167).fill(60),+ 41,+ ]);+ const expected = traces.sort(+ (a, b) =>+ a.projectId.localeCompare(b.projectId) ||+ a.due - b.due ||+ a.traceId.localeCompare(b.traceId),+ );+ expect(+ batches.flat().map((trace) => member(trace.projectId, trace.traceId)),+ ).toEqual(expected.map((trace) => member(trace.projectId, trace.traceId)));+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ });++ it("revalidates the due ID list after deletion, missing state, reactivation and new arrivals", async () => {+ const traces = Array.from(+ { length: 2_001 },+ (_, i) => `trace-${String(i).padStart(5, "0")}`,+ );+ await trackTraceBatchActivity(+ "project",+ traces.map((id) => event(id)),+ );+ const due = Date.now() - 1_000;+ await client().zadd(+ dueKey,+ ...traces.flatMap((id) => [due, member("project", id)]),+ );+ const range = client().zrange.bind(client());+ let changed = false;+ vi.spyOn(client(), "zrange").mockImplementation(async (...args) => {+ const result = await range(...args);+ if (!changed && args.includes("BYSCORE")) {+ changed = true;+ // Change Redis after the complete ID list has been read but before hydration.+ await client().zrem(dueKey, member("project", traces[0]));+ await client().hdel(stateKey, member("project", traces[0]));+ await client().hdel(stateKey, member("project", traces[1_000]));+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 1;+ await trackTraceBatchActivity("project", [+ event(traces[999]),+ event("new-after-cutoff"),+ ]);+ }+ return result;+ });+ const add = vi.spyOn(queue, "add");+ await runner().processBatch();+ expect(changed).toBe(true);+ expect(+ add.mock.calls.flatMap(([, job]) =>+ job.payload.traces.map((trace) => trace.traceId),+ ),+ ).toEqual(traces.filter((_, i) => i !== 0 && i !== 999 && i !== 1_000));+ expect(+ await client().zscore(dueKey, member("project", traces[1_000])),+ ).toBeNull();+ expect((await client().hkeys(stateKey)).sort()).toEqual(+ [+ member("project", "new-after-cutoff"),+ member("project", traces[999]),+ ].sort(),+ );+ });++ it("expires abandoned pending keys natively without another ingestion or dispatcher run", async () => {+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 300;+ await trackTraceBatchActivity("project", [event("abandoned")]);+ for (const key of [dueKey, stateKey]) {+ expect(await client().pttl(key)).toBeGreaterThan(0);+ expect(await client().pttl(key)).toBeLessThanOrEqual(300);+ }+ await vi.waitFor(async () => {+ expect(await client().exists(dueKey, stateKey)).toBe(0);+ });+ expect(await queue.getWaitingCount()).toBe(0);+ });++ it("extends native expiry for later activity and never shortens another pending trace's deadline", async () => {+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;+ await trackTraceBatchActivity("project", [event("long-idle")]);+ const initialDeadline = Number(await client().pexpiretime(dueKey));+ await vi.waitFor(() =>+ expect(Date.now()).toBeGreaterThan(initialDeadline - 60_000 + 10),+ );+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 20_000;+ await trackTraceBatchActivity("project", [event("later")]);+ const extendedDeadline = Number(await client().pexpiretime(dueKey));+ expect(extendedDeadline).toBeGreaterThan(initialDeadline);+ expect(Number(await client().pexpiretime(stateKey))).toBe(extendedDeadline);++ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 1_000;+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 1_000;+ await trackTraceBatchActivity("project", [event("short-idle")]);+ expect(Number(await client().pexpiretime(dueKey))).toBeGreaterThanOrEqual(+ extendedDeadline,+ );+ expect(Number(await client().pexpiretime(stateKey))).toBe(+ Number(await client().pexpiretime(dueKey)),+ );+ expect(await client().zcard(dueKey)).toBe(3);+ expect(await client().hlen(stateKey)).toBe(3);+ });++ it("drains and removes queued jobs without queries when reads are disabled", async () => {+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "false";+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";+ const read = vi+ .spyOn(shared, "getTraceBatchEventStream")+ .mockImplementation(async function* () {+ throw new Error("Drain mode must not query ClickHouse");+ });+ const payload = {+ traces: [+ {+ projectId: "project",+ traceId: "trace",+ minStart: 0,+ maxStart: 1,+ revision: "r",+ },+ ],+ };+ const current = await queue.add(+ QueueJobs.TraceBatch,+ {+ id: "current",+ timestamp: new Date(),+ name: QueueJobs.TraceBatch,+ payload,+ },+ { jobId: "current", removeOnComplete: { age: 3600, count: 10000 } },+ );+ const old = await queue.add(+ QueueJobs.TraceBatch,+ {+ id: "old",+ timestamp: new Date(Date.now() - 3 * 60 * 60_000),+ name: QueueJobs.TraceBatch,+ payload,+ },+ { jobId: "old", removeOnComplete: { age: 3600, count: 10000 } },+ );+ const worker = new Worker(queue.name, traceBatchQueueProcessor, {+ connection,+ prefix: getQueuePrefix(queue.name),+ });+ try {+ await vi.waitFor(async () => {+ expect(await queue.getJob(current.id!)).toBeUndefined();+ expect(await queue.getJob(old.id!)).toBeUndefined();+ });+ expect(+ await queue.getJobCounts("wait", "active", "completed", "failed"),+ ).toEqual({ wait: 0, active: 0, completed: 0, failed: 0 });+ expect(read).not.toHaveBeenCalled();+ } finally {+ await worker.close();+ }+ });++ it("expires bounded pending state during ingestion without dispatch and preserves refreshed traces", async () => {+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;+ const expired = Array.from({ length: 1_001 }, (_, i) =>+ event(`expired-${i}`),+ );+ await trackTraceBatchActivity("project", [+ ...expired,+ event("refreshed"),+ event("recent"),+ ]);+ const cutoff = Date.now() - 70_000;+ await client().zadd(+ dueKey,+ ...expired.flatMap(({ traceId }) => [cutoff, member("project", traceId)]),+ cutoff,+ member("project", "refreshed"),+ );+ await makeDue("project", "recent");+ await trackTraceBatchActivity("project", [event("refreshed", 2_000_000)]);+ expect(await client().zcard(dueKey)).toBe(3);+ expect(await client().hlen(stateKey)).toBe(3);+ expect(+ await client().hget(stateKey, member("project", "recent")),+ ).not.toBeNull();+ expect(+ Number(await client().zscore(dueKey, member("project", "refreshed"))),+ ).toBeGreaterThan(Date.now());+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.expired_traces",+ 1_000,+ );+ await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);+ expect((await client().hkeys(stateKey)).sort()).toEqual([+ member("project", "recent"),+ member("project", "refreshed"),+ ]);+ expect(await client().zcard(dueKey)).toBe(2);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.expired_traces",+ 1,+ );+ expect(await queue.getWaitingCount()).toBe(0);+ });++ it("cleans an expired backlog in bounded pages without queueing expired traces", async () => {+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;+ const expired = Array.from({ length: 10_001 }, (_, i) =>+ event(`expired-${i}`),+ );+ await trackTraceBatchActivity("project", [...expired, event("ready")]);+ await client().zadd(+ dueKey,+ ...expired.flatMap(({ traceId }) => [+ Date.now() - 70_000,+ member("project", traceId),+ ]),+ );+ await makeDue("project", "ready");+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";+ const dispatcher = runner();+ await dispatcher.processBatch();+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(+ ([name]) => name === "langfuse.trace_batch.expired_traces",+ )+ .every(([, count]) => count <= 1_000),+ ).toBe(true);+ const jobs = await queue.getJobs(["wait"]);+ expect(+ jobs.flatMap((job) =>+ job.data.payload.traces.map((trace) => trace.traceId),+ ),+ ).toEqual(["ready"]);+ expect(await client().hlen(stateKey)).toBe(0);+ expect(await client().zcard(dueKey)).toBe(0);+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(+ ([name]) => name === "langfuse.trace_batch.expired_traces",+ )+ .reduce((sum, [, value]) => sum + value, 0),+ ).toBe(10_001);+ });++ it("keeps due state on enqueue failure and retries it on the next dispatch", async () => {+ await trackTraceBatchActivity("project", [event("trace")]);+ await makeDue("project", "trace");+ vi.spyOn(queue, "add").mockRejectedValueOnce(+ new Error("enqueue unavailable"),+ );+ const dispatcher = runner();+ await dispatcher.processBatch();+ expect(await client().zcard(dueKey)).toBe(1);+ expect(await client().hlen(stateKey)).toBe(1);+ await dispatcher.processBatch();+ expect(await queue.getWaitingCount()).toBe(1);+ expect(await client().zcard(dueKey)).toBe(0);+ });++ it.each(["updated", "deleted", "expired"] as const)(+ "preserves arrivals during enqueue, including deleted/recreated state: %s",+ async (mode) => {+ await trackTraceBatchActivity("project", [event("trace")]);+ await makeDue("project", "trace");+ const add = queue.add.bind(queue);+ vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {+ const job = await add(...args);+ if (mode === "deleted")+ await client().hdel(stateKey, member("project", "trace"));+ if (mode === "expired") {+ await client().zadd(dueKey, 0, member("project", "trace"));+ await trackTraceBatchActivity("other", [event("trigger")]);+ expect(+ await client().hget(stateKey, member("project", "trace")),+ ).toBeNull();+ await client().zrem(dueKey, member("other", "trigger"));+ await client().hdel(stateKey, member("other", "trigger"));+ }+ await trackTraceBatchActivity("project", [event("trace", 2_000_000)]);+ return job;+ });+ const dispatcher = runner();+ await dispatcher.processBatch();+ expect(await queue.getWaitingCount()).toBe(1);+ expect(await client().zcard(dueKey)).toBe(1);+ const current = JSON.parse(+ (await client().hget(stateKey, member("project", "trace")))!,+ );+ const [job] = await queue.getJobs(["wait"]);+ expect(current.revision).not.toBe(job.data.payload.traces[0].revision);+ expect(current.maxStart).toBe(2_000_000);+ await makeDue("project", "trace");+ await dispatcher.processBatch();+ expect(await queue.getWaitingCount()).toBe(2);+ expect(await client().hlen(stateKey)).toBe(0);+ },+ );++ it("reuses a queued batch when its acknowledgement fails", async () => {+ await trackTraceBatchActivity("project", [event("trace")]);+ await makeDue("project", "trace");+ await trackTraceBatchActivity("other", [event("trace")]);+ await makeDue("other", "trace");+ const add = queue.add.bind(queue);+ vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {+ const job = await add(...args);+ // The next Redis operation is the post-enqueue acknowledgement.+ vi.spyOn(client(), "eval").mockRejectedValueOnce(+ new Error("ACK unavailable"),+ );+ return job;+ });+ const dispatcher = runner();+ await dispatcher.processBatch();+ expect(await queue.getWaitingCount()).toBe(1);+ expect(await client().hlen(stateKey)).toBe(2);+ await dispatcher.processBatch();+ expect(await queue.getWaitingCount()).toBe(1);+ expect(await client().hlen(stateKey)).toBe(0);+ const [job] = await queue.getJobs(["wait"]);+ expect(+ job.data.payload.traces+ .map((trace) => member(trace.projectId, trace.traceId))+ .sort(),+ ).toEqual([member("other", "trace"), member("project", "trace")]);+ });++ it("subtracts dispatcher runtime from the next delay and catches up after an overrun", async () => {+ let clock = Date.now();+ const interval = env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;+ let elapsed = Math.floor(interval / 2);+ vi.spyOn(Date, "now").mockImplementation(() => clock);+ const range = client().zrange.bind(client());+ vi.spyOn(client(), "zrange").mockImplementation(async (...args) => {+ const result = await range(...args);+ if (args.includes("BYSCORE")) clock += elapsed;+ return result;+ });+ const dispatcher = runner();+ expect(await dispatcher.processBatch()).toBe(interval - elapsed);+ elapsed = interval + 1_000;+ expect(await dispatcher.processBatch()).toBe(0);+ });++ it("bounds shutdown when an in-flight enqueue stalls and remains stopped after recovery", async () => {+ await trackTraceBatchActivity("project", [event("trace")]);+ await makeDue("project", "trace");+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const add = queue.add.bind(queue);+ vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {+ entered.resolve();+ await release.promise;+ return add(...args);+ });+ const dispatcher = runner();+ const active = dispatcher.processBatch();+ await entered.promise;+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });+ let drained = false;+ const drain = dispatcher.drain().then(() => {+ drained = true;+ });+ try {+ await vi.advanceTimersByTimeAsync(4_999);+ expect(drained).toBe(false);+ await vi.advanceTimersByTimeAsync(1);+ expect(drained).toBe(true);+ } finally {+ vi.useRealTimers();+ release.resolve();+ await Promise.all([active, drain]);+ }+ await trackTraceBatchActivity("project", [event("next")]);+ await makeDue("project", "next");+ await dispatcher.processBatch();+ expect(queue.add).toHaveBeenCalledTimes(1);+ expect(await client().hlen(stateKey)).toBe(1);+ });++ it("allows only one dispatcher and waits for its in-flight enqueue before shutdown", async () => {+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 1;+ const traces = ["trace"];+ await trackTraceBatchActivity(+ "project",+ traces.map((traceId) => event(traceId)),+ );+ await makeDue("project", ...traces);+ await trackTraceBatchActivity("other", [event("trace")]);+ await makeDue("other", "trace");+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const add = queue.add.bind(queue);+ vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {+ entered.resolve();+ await release.promise;+ return add(...args);+ });+ const first = runner();+ const active = first.processBatch();+ await entered.promise;+ await runner().processBatch();+ expect(queue.add).toHaveBeenCalledTimes(1);+ let drained = false;+ const drain = first.drain().then(() => {+ drained = true;+ });+ await Promise.resolve();+ expect(drained).toBe(false);+ release.resolve();+ await Promise.all([active, drain]);+ expect(drained).toBe(true);+ expect(await queue.getWaitingCount()).toBe(1);+ expect(await client().hlen(stateKey)).toBe(1);+ await first.processBatch();+ expect(queue.add).toHaveBeenCalledTimes(1);+ });+});diff --git a/worker/src/app.ts b/worker/src/app.tsindex 3563a43b70..f979e0e4c2 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -58,6 +58,8 @@ import { prisma } from "@langfuse/shared/src/db";import { ClickhouseReadSkipCache } from "./utils/clickhouseReadSkipCache";import { experimentCreateQueueProcessor } from "./queues/experimentQueue";import { traceDeleteProcessor } from "./queues/traceDelete";+import { traceBatchQueueProcessor } from "./queues/traceBatchQueue";+import { TraceBatchDispatcher } from "./features/traceBatching/traceBatching";import { projectDeleteProcessor } from "./queues/projectDelete";import {postHogIntegrationProcessingProcessor,@@ -136,6 +138,24 @@ ClickhouseReadSkipCache.getInstance(prisma)logger.error("Error initializing ClickhouseReadSkipCache", err);});+export let traceBatchDispatcher: TraceBatchDispatcher | null = null;+if (+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED === "true"+) {+ traceBatchDispatcher = new TraceBatchDispatcher();+ traceBatchDispatcher.start();+}++if (+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED === "true"+) {+ WorkerManager.register(QueueName.TraceBatch, traceBatchQueueProcessor, {+ concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,+ });+}+if (env.QUEUE_CONSUMER_TRACE_UPSERT_QUEUE_IS_ENABLED === "true") {// Register workers for all trace upsert queue shardsconst traceUpsertShardNames = TraceUpsertQueue.getShardNames();diff --git a/worker/src/env.ts b/worker/src/env.tsindex 4cae45be30..ac19d4a74c 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -106,6 +106,47 @@ const EnvSchema = z.object({LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED: z.enum(["true", "false"]).default("false"),+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED: z+ .enum(["true", "false"])+ .default("false"),+ LANGFUSE_TRACE_BATCH_SAMPLING_RATE: z.coerce+ .number()+ .min(0)+ .max(1)+ .default(1),+ LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED: z+ .enum(["true", "false"])+ .default("false"),+ QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED: z+ .enum(["true", "false"])+ .default("false"),+ LANGFUSE_TRACE_BATCH_READ_ENABLED: z.enum(["true", "false"]).default("false"),+ LANGFUSE_TRACE_BATCH_CONCURRENCY: z.coerce+ .number()+ .int()+ .positive()+ .default(2),+ LANGFUSE_TRACE_BATCH_MAX_SIZE: z.coerce+ .number()+ .int()+ .positive()+ .max(1_000)+ .default(60),+ LANGFUSE_TRACE_BATCH_IDLE_MS: z.coerce+ .number()+ .int()+ .positive()+ .default(600_000),+ LANGFUSE_TRACE_BATCH_PENDING_TTL_MS: z.coerce+ .number()+ .int()+ .positive()+ .default(7_200_000),+ LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS: z.coerce+ .number()+ .int()+ .positive()+ .default(30_000),LANGFUSE_SECONDARY_OTEL_INGESTION_QUEUE_ENABLED_PROJECT_IDS: z.string().optional(),diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdnew file mode 100644index 0000000000..e8e3c23207--- /dev/null+++ b/worker/src/features/traceBatching/README.md@@ -0,0 +1,117 @@+# Internal cloud trace-batch experiment++This read-only experiment measures full-event reads for idle traces. It requires+`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` and explicit opt-in. All four enablement+flags below default to `false`; an ordinary release needs no infrastructure+changes. These internal controls are intentionally absent from env templates.++## Controls++| Variable | Default | Role |+| --------------------------------------------- | --------- | ------------------------------------------------------------- |+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 1,000. |+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |++Producer, dispatcher and consumer can run independently. Sampling reuses the+evaluator's deterministic trace-ID sampler; every observation of the same trace+gets the same admission decision. Dispatch and consumption do not resample.+Cloud guards cover ingestion, dispatcher execution, worker registration and+processing, and the shared queue accessor, including admin queue inspection.++## Redis state and expiry++- `{trace-batch}:due`: sorted set, with JSON `[projectId, traceId]` members and+ Redis server time plus idle delay as the score.+- `{trace-batch}:state`: hash with the same members and JSON+ `{minStart, maxStart, revision}` values. Bounds describe observed event start+ times; revision protects newer ingestion from an older dispatch acknowledgment.+- `{trace-batch}:dispatcher`: renewable dispatcher lease.++No event payloads, per-trace byte counts or observation counts are stored here.+Atomic Lua updates refresh bounds, revision and readiness. Each admitted intake+sets a shared absolute expiry on both pending keys with `PEXPIREAT`, preserving+any later existing expiry using `PEXPIRETIME` (Redis 7+). With no further intake,+both keys expire after the retention period even if every application stops;+idle time is not added to this deadline.++This is a whole-key inactivity backstop, not native per-member expiry. During+continuous intake, bounded Lua cleanup removes members whose due time is older+than retention; ingestion and dispatcher both perform cleanup.++## Dispatch and reads++One dispatcher holds the renewable lease. It snapshots ready IDs, sorts them by+project, then hydrates and revalidates state in chunks of 1,000. Jobs contain up+to the configured trace cap; project leftovers can share a job. The dispatcher+enqueues before acknowledging matching revisions, so concurrent intake stays+pending and failed enqueueing can be retried. Stable job IDs limit duplicates.++The worker streams full `events_full` payloads and records batch observation,+trace, project and logical payload-byte metrics. The `input_bytes`, `output_bytes`+and `metadata_bytes` distributions under `langfuse.trace_batch` each report one+total per completed batch read; `io_metadata_bytes` retains their combined total.+Sizes count UTF-8 bytes, including metadata keys and values, excluding JSON+transport framing/escaping and compression. They do not measure ClickHouse bytes+scanned or per-trace ingestion size; retries can record another batch sample.+Exact `(projectId, traceId)`+pairs protect tenant boundaries; project, trace hash and a shared batch time+window prune reads. `TRACE_QUERY_BUFFER_MS` pads the earliest and latest recorded+start times by two minutes each. This fixed query margin can include observations+outside the recorded bounds; it neither waits for arrivals nor guarantees trace+completeness. Query limits are fixed at two threads and 30 seconds with+timeouts failing the job. This baseline has no hash-locality grouping,+per-trace windows, query tuning controls or allocation based on trace size.++## Monitor dispatcher memory++The baseline intentionally snapshots the complete ready cohort. Batch size limits+jobs, not snapshot memory. Retain this behavior for the initial experiment and+watch memory alongside backlog growth before deciding whether to add paging.++The worker enables Datadog Node runtime metrics. In Metrics Explorer, filter to+the experiment environment and worker service, and inspect the maximum per+worker/container rather than a fleet average:++- `runtime.node.mem.rss`: total process memory; compare with the container limit.+- `runtime.node.mem.heap_used`: JavaScript heap in use.+- `langfuse.trace_batch.snapshot_size`, `pending_traces`, `ready_traces` and+ `oldest_due_age_ms`: snapshot and backlog growth.+- `langfuse.periodic_runner.duration_ms` and `completed`, filtered by+ `runner:trace_batch_dispatcher`: run duration and outcome.++Runtime memory is shared with other work in the process, so correlation with+dispatch runs does not isolate the dispatcher's allocations. Periodic sampling+can miss short peaks, and `snapshot_size` is emitted only after parsing/sorting+finishes. Check worker restarts/OOM events as well. Verify live metric delivery+and available instance tags in Datadog before relying on these signals.++## Stop and drain++Disable ingestion and dispatch to stop new work. Leave the consumer enabled and+set the read flag to `false` to complete and remove jobs without querying or+parsing their payloads. Jobs at least two hours old are also discarded on pickup.+Reads already in progress finish under their query timeout.++On process shutdown, the dispatcher stops scheduling and waits up to five seconds+for its active dispatch. If Redis stalls, it logs a warning and lets the remaining+shutdown steps proceed. This timeout does not cancel an already issued Redis command.++BullMQ waiting jobs have no autonomous TTL: disabling all consumers leaves them+queued until consumption resumes. Draining requires an active, unpaused worker;+delayed jobs become eligible later. Normal completed jobs retain IDs for up to+one hour / 10,000 jobs, and failures retain up to 1,000 jobs under BullMQ's+lazy retention rules. Drain and expired jobs are removed immediately on success.++For an already enabled cloud experiment, explicitly set the new read flag to+`true` when deploying this version if reads should continue; leaving it unset+drains instead. Upgrade all producers before relying on native expiry: older+producers do not install or refresh that TTL. Locality, query controls and load+test artifacts belong in separate follow-up changes.diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsnew file mode 100644index 0000000000..a2dfa70155--- /dev/null+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -0,0 +1,409 @@+import { createHash, randomUUID } from "node:crypto";+import {+ logger,+ QueueJobs,+ redis,+ recordDistribution,+ recordGauge,+ recordIncrement,+ TraceBatchTraceSchema,+ TraceBatchQueue,+ type QueueName,+ type TQueueJobTypes,+} from "@langfuse/shared/src/server";+import { env } from "../../env";+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";+import {+ getDeterministicSamplingValue,+ shouldSampleEvaluation,+} from "../evaluation/deterministicSampling";++// The shared client applies REDIS_KEY_PREFIX. Both keys must occupy one+// Redis Cluster slot for the atomic update, snapshot, and acknowledgement.+const DUE_KEY = "{trace-batch}:due";+const STATE_KEY = "{trace-batch}:state";+const CHUNK_SIZE = 1_000;+type PendingTrace = {+ member: string;+ due: number;+ trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];+};++// Retain traces for a bounded time after readiness. Reuse the due index so+// cleanup needs no full hash scan. Whole-key expiry is an inactivity backstop.+const EXPIRE_PENDING_SCRIPT = `+ local function expirePending(now, retention, limit)+ local expired = redis.call('ZRANGE', KEYS[1], '-inf', now - retention,+ 'BYSCORE', 'LIMIT', 0, limit)+ for _, member in ipairs(expired) do+ redis.call('ZREM', KEYS[1], member)+ redis.call('HDEL', KEYS[2], member)+ end+ return #expired+ end+`;++const TRACK_SCRIPT = `+ ${EXPIRE_PENDING_SCRIPT}+ local clock = redis.call('TIME')+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)+ for i = 3, #ARGV, 4 do+ local member = ARGV[i]+ local minStart = tonumber(ARGV[i + 1])+ local maxStart = tonumber(ARGV[i + 2])+ local previous = redis.call('HGET', KEYS[2], member)+ if previous then+ local state = cjson.decode(previous)+ minStart = math.min(minStart, state.minStart)+ maxStart = math.max(maxStart, state.maxStart)+ end+ redis.call('HSET', KEYS[2], member, cjson.encode({+ minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3]+ }))+ redis.call('ZADD', KEYS[1], now + tonumber(ARGV[1]), member)+ end+ local retention = tonumber(ARGV[2])+ local expired = expirePending(now, retention, ${CHUNK_SIZE})+ -- Bound abandoned state from the last admitted activity, independent of idle.+ -- A shorter policy on another writer must not shorten an existing expiry.+ local expiresAt = math.max(now + retention,+ redis.call('PEXPIRETIME', KEYS[1]), redis.call('PEXPIRETIME', KEYS[2]))+ redis.call('PEXPIREAT', KEYS[1], expiresAt)+ redis.call('PEXPIREAT', KEYS[2], expiresAt)+ return expired+`;++// Cleanup stays bounded even when the dispatcher has accumulated a backlog.+// Keep the first Redis timestamp as the cutoff throughout this run.+const SNAPSHOT_SCRIPT = `+ ${EXPIRE_PENDING_SCRIPT}+ local clock = redis.call('TIME')+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)+ local retention = tonumber(ARGV[1])+ local cutoff = tonumber(ARGV[2]) or now+ local expired = expirePending(now, retention, ${CHUNK_SIZE})+ local pending = redis.call('ZCARD', KEYS[1])+ local ready = math.max(0, redis.call('ZCOUNT', KEYS[1], '-inf', cutoff)+ - redis.call('ZCOUNT', KEYS[1], '-inf', now - retention))+ local first = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')+ return {now, cutoff, pending, ready, first[2] or tostring(now), expired}+`;++// Fetch state only while it is still due for this run. Checking the score and+// reading the revision atomically prevents dispatching freshly reactivated work.+const HYDRATE_SCRIPT = `+ local clock = redis.call('TIME')+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)+ local result = {}+ for i = 3, #ARGV do+ local member = ARGV[i]+ local due = tonumber(redis.call('ZSCORE', KEYS[1], member))+ if due and due <= tonumber(ARGV[1]) and due > now - tonumber(ARGV[2]) then+ local state = redis.call('HGET', KEYS[2], member)+ if state then+ table.insert(result, member)+ table.insert(result, state)+ table.insert(result, tostring(due))+ else+ redis.call('ZREM', KEYS[1], member)+ end+ end+ end+ return result+`;++const ACKNOWLEDGE_SCRIPT = `+ local removed = 0+ for i = 1, #ARGV, 2 do+ local state = redis.call('HGET', KEYS[2], ARGV[i])+ if state and cjson.decode(state).revision == ARGV[i + 1] then+ redis.call('ZREM', KEYS[1], ARGV[i])+ redis.call('HDEL', KEYS[2], ARGV[i])+ removed = removed + 1+ end+ end+ return removed+`;++export async function trackTraceBatchActivity(+ projectId: string,+ events: { traceId: string; startTimeISO: string }[],+): Promise<void> {+ if (+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED !== "true"+ )+ return;++ const bounds = new Map<string, { minStart: number; maxStart: number }>();+ for (const event of events) {+ const start = Date.parse(event.startTimeISO);+ if (!Number.isFinite(start)) continue;+ const previous = bounds.get(event.traceId);+ bounds.set(event.traceId, {+ minStart: Math.min(previous?.minStart ?? start, start),+ maxStart: Math.max(previous?.maxStart ?? start, start),+ });+ }+ if (bounds.size === 0) return;++ try {+ const samplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;+ // Sample by trace ID so later observations and retries keep the same decision.+ // Dispatcher and consumer process admitted work without resampling.+ const entries = [...bounds].filter(+ ([traceId]) =>+ samplingRate === 1 ||+ (samplingRate > 0 &&+ shouldSampleEvaluation({+ samplingValue: getDeterministicSamplingValue(traceId),+ samplingRate,+ })),+ );+ recordGauge("langfuse.trace_batch.sampling_rate", samplingRate);+ // Counts distinct trace IDs per ingestion batch, not globally unique traces.+ recordIncrement("langfuse.trace_batch.sampling_decisions", entries.length, {+ decision: "selected",+ });+ recordIncrement(+ "langfuse.trace_batch.sampling_decisions",+ bounds.size - entries.length,+ { decision: "excluded" },+ );+ if (entries.length === 0) return;+ if (!redis) throw new Error("Trace batching requires Redis");+ // Bounded scripts keep ingestion from monopolizing the global Redis slot.+ for (let offset = 0; offset < entries.length; offset += CHUNK_SIZE) {+ const chunk = entries.slice(offset, offset + CHUNK_SIZE);+ const expired = Number(+ await redis.eval(+ TRACK_SCRIPT,+ 2,+ DUE_KEY,+ STATE_KEY,+ env.LANGFUSE_TRACE_BATCH_IDLE_MS,+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,+ ...chunk.flatMap(([traceId, state]) => [+ JSON.stringify([projectId, traceId]),+ state.minStart,+ state.maxStart,+ // Unique even after dispatch deletes and a later arrival recreates state.+ randomUUID(),+ ]),+ ),+ );+ recordIncrement("langfuse.trace_batch.expired_traces", expired);+ recordIncrement("langfuse.trace_batch.tracked_traces", chunk.length);+ }+ } catch (error) {+ // This optional load experiment must not fail normal event ingestion.+ recordIncrement("langfuse.trace_batch.tracking_errors", 1);+ logger.error("Failed to track trace batch activity", { error });+ }+}++export class TraceBatchDispatcher extends PeriodicExclusiveRunner {+ private stopping = false;+ private activeDispatch: Promise<void> | null = null;++ constructor() {+ super({+ name: "TraceBatchDispatcher",+ metricName: "trace_batch_dispatcher",+ lockKey: "{trace-batch}:dispatcher",+ lockTtlSeconds: 60,+ onUnavailable: "fail",+ });+ }++ protected get defaultIntervalMs(): number {+ return env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;+ }++ public override stop(): void {+ this.stopping = true;+ super.stop();+ }++ public async drain(): Promise<void> {+ this.stop();+ if (!this.activeDispatch) return;+ let timeout: ReturnType<typeof setTimeout> | undefined;+ try {+ await Promise.race([+ this.activeDispatch,+ new Promise<void>((resolve) => {+ timeout = setTimeout(() => {+ logger.warn(+ "Trace batch dispatcher drain timed out after 5 seconds; continuing shutdown",+ );+ resolve();+ }, 5_000);+ }),+ ]);+ } finally {+ clearTimeout(timeout);+ }+ }++ protected async execute(): Promise<number | void> {+ if (+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED !== "true" ||+ this.stopping ||+ this.activeDispatch+ )+ return;+ const startedAt = Date.now();+ const dispatch = this.withLock(() => this.dispatchDue()).then(() => {});+ this.activeDispatch = dispatch;+ try {+ await dispatch;+ } finally {+ this.activeDispatch = null;+ }+ // PeriodicRunner waits this delay after completion. Account for work time+ // to maintain the configured start cadence without overlapping runs.+ return Math.max(0, this.defaultIntervalMs - (Date.now() - startedAt));+ }++ private async dispatchDue(): Promise<void> {+ if (!redis) throw new Error("Trace batching requires Redis");+ const queue = TraceBatchQueue.getInstance();+ if (!queue) throw new Error("Trace batch queue is unavailable");+ let cutoff = "";+ let now = 0;+ while (!this.stopping) {+ await this.extendLockOnProgress();+ const snapshot = (await redis.eval(+ SNAPSHOT_SCRIPT,+ 2,+ DUE_KEY,+ STATE_KEY,+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,+ cutoff,+ )) as (string | number)[];+ const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);+ now = clock;+ cutoff = String(snapshot[1]);+ recordIncrement("langfuse.trace_batch.expired_traces", expired);+ recordGauge("langfuse.trace_batch.pending_traces", pending);+ recordGauge("langfuse.trace_batch.ready_traces", ready);+ recordGauge(+ "langfuse.trace_batch.oldest_due_age_ms",+ Math.max(0, now - oldest),+ );+ if (expired < CHUNK_SIZE) break;+ }+ if (this.stopping) return;++ // One range read captures all due IDs without cursor races or loading every+ // trace's state. Memory and the Redis response size grow with the due cohort.+ const members = (+ await redis.zrange(+ DUE_KEY,+ `(${now - env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS}`,+ cutoff,+ "BYSCORE",+ )+ ).map((member) => ({+ member,+ projectId: (JSON.parse(member) as [string, string])[0],+ }));+ // Stable sorting keeps Redis readiness order within each project.+ members.sort((a, b) =>+ a.projectId < b.projectId ? -1 : a.projectId > b.projectId ? 1 : 0,+ );+ recordDistribution("langfuse.trace_batch.snapshot_size", members.length);++ const enqueue = async (batch: PendingTrace[]) => {+ await this.extendLockOnProgress(true);+ if (this.stopping) return;+ const traces = batch.map(({ trace }) => trace);+ const id = createHash("sha256")+ .update(JSON.stringify(traces))+ .digest("hex");+ // Stable IDs reduce duplicate delivery if enqueue succeeds but ACK fails.+ // Reads remain retry-safe even if regrouping produces a different job ID.+ await queue.add(+ QueueJobs.TraceBatch,+ {+ id,+ timestamp: new Date(),+ name: QueueJobs.TraceBatch,+ payload: { traces },+ },+ { jobId: id },+ );+ recordDistribution("langfuse.trace_batch.size", batch.length);+ recordDistribution(+ "langfuse.trace_batch.project_count",+ new Set(traces.map((trace) => trace.projectId)).size,+ );+ recordIncrement("langfuse.trace_batch.dispatched_traces", batch.length, {+ batch_kind: batch.length === 1 ? "singleton" : "multi",+ });+ for (const entry of batch) {+ recordDistribution(+ "langfuse.trace_batch.due_lag_ms",+ Math.max(0, Date.now() - entry.due),+ );+ }+ const removed = Number(+ await redis!.eval(+ ACKNOWLEDGE_SCRIPT,+ 2,+ DUE_KEY,+ STATE_KEY,+ ...batch.flatMap(({ member, trace }) => [member, trace.revision]),+ ),+ );+ recordIncrement(+ "langfuse.trace_batch.reactivated_traces",+ batch.length - removed,+ );+ };++ let batch: PendingTrace[] = [];+ for (+ let offset = 0;+ offset < members.length && !this.stopping;+ offset += CHUNK_SIZE+ ) {+ await this.extendLockOnProgress();+ const candidates = members.slice(offset, offset + CHUNK_SIZE);+ const hydrated = (await redis.eval(+ HYDRATE_SCRIPT,+ 2,+ DUE_KEY,+ STATE_KEY,+ cutoff,+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,+ ...candidates.map(({ member }) => member),+ )) as string[];+ recordIncrement(+ "langfuse.trace_batch.skipped_traces",+ candidates.length - hydrated.length / 3,+ );+ for (let i = 0; i < hydrated.length; i += 3) {+ if (this.stopping) return;+ const member = hydrated[i];+ const [projectId, traceId] = JSON.parse(member) as [string, string];+ batch.push({+ member,+ due: Number(hydrated[i + 2]),+ trace: TraceBatchTraceSchema.parse({+ ...JSON.parse(hydrated[i + 1]),+ projectId,+ traceId,+ }),+ });+ if (batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {+ await enqueue(batch);+ batch = [];+ }+ }+ }+ if (!this.stopping && batch.length > 0) await enqueue(batch);+ }+}diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.tsindex 7a91be20b8..1b65d2ce64 100644--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts@@ -1,4 +1,18 @@-import { describe, it, expect } from "vitest";+import { describe, it, expect, vi } from "vitest";+import { type Job } from "bullmq";+import {+ getS3EventStorageClient,+ OtelIngestionProcessor,+ QueueJobs,+ type QueueName,+ recordDistribution,+ type TQueueJobTypes,+} from "@langfuse/shared/src/server";+import { env } from "../../env";+import { IngestionService } from "../../services/IngestionService";+import { ClickhouseWriter } from "../../services/ClickhouseWriter";+import { fetchObservationEvalRules } from "../../features/evaluation/observationEval";+import { trackTraceBatchActivity } from "../../features/traceBatching/traceBatching";import {batchContainsLangfuseScope,checkHeaderBasedDirectWrite,@@ -6,11 +20,148 @@ import {getSdkInfoFromResourceSpans,isLangfuseSdkTraffic,isOrgPastOtelDirectWriteCutoff,+ otelIngestionQueueProcessorBuilder,resolveOtelWritePath,shouldProcessLegacyOtelMedia,type SdkInfo,} from "../otelIngestionQueue";+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),+ getS3EventStorageClient: vi.fn(),+ recordDistribution: vi.fn(),+}));+vi.mock(+ "../../features/evaluation/observationEval",+ async (importOriginal) => ({+ ...(await importOriginal<+ typeof import("../../features/evaluation/observationEval")+ >()),+ fetchObservationEvalRules: vi.fn(),+ }),+);+vi.mock("../../features/traceBatching/traceBatching", () => ({+ trackTraceBatchActivity: vi.fn(),+}));++describe("direct-v4 trace batch tracking", () => {+ it("tracks accepted writes together and excludes failed writes, disabled intake, and legacy writes", async () => {+ const original = {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,+ LANGFUSE_TRACE_BATCH_INGESTION_ENABLED:+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED,+ LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,+ LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED:+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,+ };+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";+ env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED = "false";++ try {+ vi.mocked(getS3EventStorageClient).mockReturnValue({+ download: vi.fn().mockResolvedValue("[]"),+ } as unknown as ReturnType<typeof getS3EventStorageClient>);+ vi.mocked(fetchObservationEvalRules).mockResolvedValue([]);+ vi.spyOn(ClickhouseWriter, "getInstance").mockReturnValue(+ {} as ClickhouseWriter,+ );+ vi.spyOn(+ OtelIngestionProcessor.prototype,+ "processToIngestionEvents",+ ).mockResolvedValue([]);+ const inputs = [+ { traceId: "accepted", spanId: "first" },+ { traceId: "accepted", spanId: "second" },+ { traceId: "write-failed", spanId: "third" },+ { traceId: "conversion-failed", spanId: "fourth" },+ ].map((entry, index) => ({+ ...entry,+ projectId: "project",+ startTimeISO: new Date(1_000 + index * 1_000).toISOString(),+ }));+ vi.spyOn(+ OtelIngestionProcessor.prototype,+ "processToEvent",+ ).mockReturnValue(+ inputs as ReturnType<OtelIngestionProcessor["processToEvent"]>,+ );+ vi.spyOn(+ IngestionService.prototype,+ "createEventRecord",+ ).mockImplementation(async (input) => {+ if (input.traceId === "conversion-failed") throw new Error("invalid");+ return { span_id: input.spanId } as Awaited<+ ReturnType<IngestionService["createEventRecord"]>+ >;+ });+ const write = vi+ .spyOn(IngestionService.prototype, "writeEventRecord")+ .mockImplementation(async (record) => {+ if (record.span_id === "third") throw new Error("write rejected");+ });+ const processor = otelIngestionQueueProcessorBuilder(false);+ const job = {+ data: {+ id: "ingestion",+ timestamp: new Date(),+ name: QueueJobs.OtelIngestionJob,+ payload: {+ data: { fileKey: "synthetic.json" },+ authCheck: {+ validKey: true,+ scope: {+ projectId: "project",+ orgId: "org",+ accessLevel: "project",+ },+ },+ ingestionVersion: "4",+ },+ },+ } as Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>;++ await processor(job, undefined);+ expect(write).toHaveBeenCalledTimes(3);+ expect(trackTraceBatchActivity).toHaveBeenCalledExactlyOnceWith(+ "project",+ inputs+ .slice(0, 2)+ .map(({ traceId, startTimeISO }) => ({ traceId, startTimeISO })),+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.ingestion_trace_count",+ 1,+ );++ vi.mocked(trackTraceBatchActivity).mockClear();+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;+ await processor(job, undefined);+ expect(trackTraceBatchActivity).not.toHaveBeenCalled();+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";+ await processor(job, undefined);+ expect(trackTraceBatchActivity).not.toHaveBeenCalled();++ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";+ write.mockRejectedValue(new Error("all writes rejected"));+ await processor(job, undefined);+ expect(trackTraceBatchActivity).not.toHaveBeenCalled();++ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";+ write.mockClear();+ await processor(job, undefined);+ expect(write).not.toHaveBeenCalled();+ expect(trackTraceBatchActivity).not.toHaveBeenCalled();+ } finally {+ Object.assign(env, original);+ vi.restoreAllMocks();+ vi.clearAllMocks();+ }+ });+});+describe("isOrgPastOtelDirectWriteCutoff", () => {const cutoff = "2026-08-06";const inScope = {diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsnew file mode 100644index 0000000000..cdb29bed53--- /dev/null+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -0,0 +1,254 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { type Job } from "bullmq";+import {+ getTraceBatchEventStream,+ QueueJobs,+ type QueueName,+ recordDistribution,+ type TQueueJobTypes,+} from "@langfuse/shared/src/server";+import { env } from "../../env";+import { traceBatchQueueProcessor } from "../traceBatchQueue";++vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),+ getTraceBatchEventStream: vi.fn(),+ recordDistribution: vi.fn(),+}));++const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;+const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;+beforeEach(() => {+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+});+afterEach(() => {+ env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;+ vi.restoreAllMocks();+ vi.clearAllMocks();+});++describe("trace batch queue", () => {+ it("discards self-hosted jobs before parsing even when reads are enabled", async () => {+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;+ const job = {+ opts: {},+ data: { incompatiblePayload: true },+ } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({+ discarded: "not_cloud",+ });+ expect(getTraceBatchEventStream).not.toHaveBeenCalled();+ expect(recordDistribution).not.toHaveBeenCalled();+ expect(job.opts.removeOnComplete).toBe(true);+ });++ it("discards an expired batch before querying, including on retry", async () => {+ const job = {+ opts: {},+ data: {+ id: "expired",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(Date.now() - 3 * 60 * 60_000).toISOString(),+ payload: {+ traces: [+ {+ projectId: "project",+ traceId: "trace",+ minStart: 0,+ maxStart: 1,+ revision: "r",+ },+ ],+ },+ },+ } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ for (let attempt = 0; attempt < 2; attempt++) {+ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({+ discarded: "expired",+ });+ }+ expect(getTraceBatchEventStream).not.toHaveBeenCalled();+ expect(recordDistribution).not.toHaveBeenCalled();+ expect(job.opts.removeOnComplete).toBe(true);+ });++ it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {+ const ingestionEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;+ const dispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "false";+ try {+ const payload = {+ traces: [+ ["project", "trace-a"],+ ["project", "trace-b"],+ ["other", "trace-a"],+ ["project", "trace-missing"],+ ["absent", "trace-missing"],+ ].map(([projectId, traceId]) => ({+ projectId,+ traceId,+ minStart: 1_000,+ maxStart: 2_000,+ revision: "revision",+ })),+ };+ const job = {+ data: {+ id: "batch",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(),+ payload,+ },+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ let yieldedRows = 0;+ vi.mocked(getTraceBatchEventStream).mockImplementation(+ async function* () {+ for (const [projectId, traceId] of [+ ["project", "trace-a"],+ ["project", "trace-a"],+ ["project", "trace-b"],+ ["other", "trace-a"],+ ]) {+ yieldedRows++;+ yield {+ project_id: projectId,+ trace_id: traceId,+ span_id: `span-${yieldedRows}`,+ parent_span_id: null,+ start_time: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000000",+ type: "GENERATION",+ name: "generation",+ input: "hello",+ output: "世界",+ metadata: { é: "界🙂" },+ tool_definitions: {},+ tool_calls: [],+ tool_call_names: [],+ };+ }+ },+ );++ for (let attempt = 0; attempt < 2; attempt++) {+ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual(+ {+ observationCount: 4,+ traceCount: 3,+ projectCount: 2,+ inputBytes: 20,+ outputBytes: 24,+ metadataBytes: 36,+ ioMetadataBytes: 80,+ },+ );+ }+ expect(getTraceBatchEventStream).toHaveBeenCalledTimes(2);+ expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload);+ expect(yieldedRows).toBe(8);+ for (const [name, bytes] of [+ ["input_bytes", 20],+ ["output_bytes", 24],+ ["metadata_bytes", 36],+ ["io_metadata_bytes", 80],+ ] as const) {+ expect(+ vi+ .mocked(recordDistribution)+ .mock.calls.filter(+ ([metric]) => metric === `langfuse.trace_batch.${name}`,+ )+ .map(([, value]) => value),+ ).toEqual([bytes, bytes]);+ }+ expect(+ vi+ .mocked(recordDistribution)+ .mock.calls.filter(+ ([name]) => name === "langfuse.trace_batch.found_project_count",+ )+ .map(([, value]) => value),+ ).toEqual([2, 2]);+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.missing_trace_count",+ 2,+ );+ } finally {+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = ingestionEnabled;+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = dispatcherEnabled;+ }+ });++ it.each([true, false])(+ "normalizes persisted single-project jobs and counts returned projects (has rows: %s)",+ async (hasRows) => {+ const trace = {+ traceId: "trace",+ minStart: 1_000,+ maxStart: 2_000,+ revision: "revision",+ };+ const job = {+ data: {+ id: "legacy-batch",+ name: QueueJobs.TraceBatch,+ timestamp: new Date().toISOString(),+ payload: { projectId: "project", traces: [trace] },+ },+ } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ vi.mocked(getTraceBatchEventStream).mockImplementation(+ async function* () {+ if (!hasRows) return;+ yield {+ project_id: "project",+ trace_id: "trace",+ span_id: "span",+ parent_span_id: null,+ start_time: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000000",+ type: "GENERATION",+ name: "generation",+ input: "hello",+ output: "world",+ metadata: {},+ tool_definitions: {},+ tool_calls: [],+ tool_call_names: [],+ };+ },+ );++ await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({+ observationCount: Number(hasRows),+ traceCount: Number(hasRows),+ projectCount: Number(hasRows),+ inputBytes: hasRows ? 5 : 0,+ outputBytes: hasRows ? 5 : 0,+ metadataBytes: 0,+ ioMetadataBytes: hasRows ? 10 : 0,+ });+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.input_bytes",+ hasRows ? 5 : 0,+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.output_bytes",+ hasRows ? 5 : 0,+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.metadata_bytes",+ 0,+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.found_project_count",+ Number(hasRows),+ );+ expect(getTraceBatchEventStream).toHaveBeenCalledWith({+ traces: [{ ...trace, projectId: "project" }],+ });+ },+ );+});diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.tsindex ae71281f71..5d4377ef0a 100644--- a/worker/src/queues/otelIngestionQueue.ts+++ b/worker/src/queues/otelIngestionQueue.ts@@ -36,6 +36,7 @@ import {v4WritesToLegacyTables,} from "../env";import { IngestionService } from "../services/IngestionService";+import { trackTraceBatchActivity } from "../features/traceBatching/traceBatching";import { prisma } from "@langfuse/shared/src/db";import { ClickhouseWriter } from "../services/ClickhouseWriter";import {@@ -830,6 +831,8 @@ export const otelIngestionQueueProcessorBuilder = (? createObservationEvalSchedulerDeps(): null;+ const traceBatchEvents: { traceId: string; startTimeISO: string }[] = [];+await Promise.all(// Process each event independentlyeventInputs.map(async (eventInput) => {@@ -880,6 +883,15 @@ export const otelIngestionQueueProcessorBuilder = (if (shouldWriteToEventsTable) {try {await ingestionService.writeEventRecord(eventRecord);+ if (+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED === "true"+ ) {+ traceBatchEvents.push({+ traceId: eventInput.traceId,+ startTimeISO: eventInput.startTimeISO,+ });+ }} catch (error) {traceException(error);logger.error(@@ -890,6 +902,15 @@ export const otelIngestionQueueProcessorBuilder = (}}),);++ if (traceBatchEvents.length > 0) {+ recordDistribution(+ "langfuse.trace_batch.ingestion_trace_count",+ new Set(traceBatchEvents.map((event) => event.traceId)).size,+ );+ // The writer has accepted these records; its flush completes separately.+ await trackTraceBatchActivity(projectId, traceBatchEvents);+ }} catch (e) {const fileKey = job.data.payload.data.fileKey;if (e instanceof ForbiddenError) {diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsnew file mode 100644index 0000000000..522b1c4278--- /dev/null+++ b/worker/src/queues/traceBatchQueue.ts@@ -0,0 +1,90 @@+import { type Processor } from "bullmq";+import {+ getTraceBatchEventStream,+ recordDistribution,+ recordIncrement,+ TraceBatchEventSchema,+ type QueueName,+ type TQueueJobTypes,+} from "@langfuse/shared/src/server";+import { env } from "../env";++const JOB_MAX_AGE_MS = 2 * 60 * 60_000;++export const traceBatchQueueProcessor: Processor<+ TQueueJobTypes[QueueName.TraceBatch]+> = async (job) => {+ const disabledReason = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION+ ? "not_cloud"+ : env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true"+ ? "reads_disabled"+ : undefined;+ if (disabledReason) {+ // BullMQ reads these options after the processor returns, then removes atomically.+ job.opts.removeOnComplete = true;+ recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {+ reason: disabledReason,+ });+ return { discarded: disabledReason };+ }+ const event = TraceBatchEventSchema.parse(job.data);+ if (Date.now() - event.timestamp.getTime() >= JOB_MAX_AGE_MS) {+ job.opts.removeOnComplete = true;+ recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {+ reason: "expired",+ });+ return { discarded: "expired" };+ }+ const batch = event.payload;+ const foundTraces = new Set<string>();+ const foundProjects = new Set<string>();+ let observationCount = 0;+ let inputBytes = 0;+ let outputBytes = 0;+ let metadataBytes = 0;++ for await (const event of getTraceBatchEventStream(batch)) {+ observationCount++;+ foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));+ foundProjects.add(event.project_id);+ // Logical UTF-8 payload size, excluding JSON transport and compression.+ inputBytes += Buffer.byteLength(event.input);+ outputBytes += Buffer.byteLength(event.output);+ for (const [key, value] of Object.entries(event.metadata)) {+ metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);+ }+ }++ recordDistribution(+ "langfuse.trace_batch.observation_count",+ observationCount,+ );+ recordDistribution(+ "langfuse.trace_batch.found_trace_count",+ foundTraces.size,+ );+ const ioMetadataBytes = inputBytes + outputBytes + metadataBytes;+ recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);+ recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);+ recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);+ recordDistribution("langfuse.trace_batch.io_metadata_bytes", ioMetadataBytes);+ recordDistribution(+ "langfuse.trace_batch.found_project_count",+ foundProjects.size,+ );+ recordDistribution(+ "langfuse.trace_batch.missing_trace_count",+ batch.traces.length - foundTraces.size,+ );++ // Retries can repeat this read; observation payloads never enter job results.+ return {+ observationCount,+ traceCount: foundTraces.size,+ projectCount: foundProjects.size,+ inputBytes,+ outputBytes,+ metadataBytes,+ ioMetadataBytes,+ };+};diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex 3717e49d6f..812fe1ce59 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -24,6 +24,7 @@ import {queueMetricsRunner,monitorRunners,inAppAgentDlqRetryRunner,+ traceBatchDispatcher,} from "../app";export const onShutdown: NodeJS.SignalsListener = async (signal) => {@@ -50,6 +51,9 @@ const runDrainAndClose = async () => {server?.close();logger.info("Server has been closed.");+ // Give in-flight dispatch up to five seconds before continuing shutdown.+ await traceBatchDispatcher?.drain();+// Stop batch project cleanersfor (const cleaner of batchProjectCleaners) {cleaner.stop();