🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(trace-batching): reprocess late arrivals with sharded readiness (#18602)
Lysander Kiesel6069e090ee9d
diff --git a/.env.dev.example b/.env.dev.exampleindex 72ab869c2d..1346352018 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -93,6 +93,10 @@ NEXTAUTH_SECRET="secret"# LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES=104857600# Unset idle is 2 minutes when NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=DEV, else 10 minutes.LANGFUSE_TRACE_BATCH_IDLE_MS=120000+# Internal cloud readiness retention (at most 48h) and sharding; drain before changing.+# LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS=86400000+# Optional override; unset uses 16 on Redis Cluster and 1 on standalone/Sentinel.+# LANGFUSE_TRACE_BATCH_SHARD_COUNT=16# Summaries and embeddings use Bedrock in LANGFUSE_AI_AWS_BEDROCK_REGION.# Optional local profile: LANGFUSE_AI_FEATURES_AWS_PROFILE (AWS_PROFILE wins).# Postgres, ClickHouse, and Redis are the same local services as the rest of the stack.diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex 9cfdd34018..0042823fef 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -30,6 +30,8 @@- Repository layer: `src/server/repositories/*`- Queue payload schemas: `src/server/queues.ts`- Queue helpers: `src/server/redis/*`+ `getShardIndex` from `src/server/redis/sharding.ts` is exported by the server+ barrel; hash the unit identity when a project can dominate traffic.- Internal trace-batch queue: `src/server/redis/traceBatch.ts` (cloud-gated);payloads in `src/server/queues.ts` accept persisted single-project jobs.Full-event streaming reads live in `src/server/repositories/trace-batch.ts`;diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex d7a3bffdf4..b17fad518c 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -110,6 +110,7 @@ export * from "./redis/monitorQueue";export * from "./redis/inAppAgentRunQueue";export * from "./redis/traceDelete";export * from "./redis/traceBatch";+export * from "./redis/sharding";export * from "./redis/projectDelete";export * from "./redis/scoreDelete";export * from "./redis/datasetDelete";diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex fa762dc2d6..b78cd131eb 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -50,6 +50,26 @@`src/features/traceBatching/TraceBatchMetricsRunner.ts` collects bounded queueand Redis snapshots independently of dispatch/consumption when either role isenabled. Global snapshot gauges must not be summed across worker reporters.+ Readiness uses project/trace-hashed shards with one dispatcher lease; all Lua+ keys share a shard hash tag. Defaults are 16 shards on Redis Cluster and 1 on+ standalone/Sentinel, with an explicit count override. Redis resizing does not+ change the count. Hourly first-seen hashes retain packed minimum and+ maximum event starts and cumulative size estimates after acknowledgement for late arrivals;+ history retention defaults to 24 hours and is capped at 48 hours. Pending state+ carries the bucket pointer. Pending expiry is due plus recovery grace, including+ the whole-key backstop. Tracking uses the project's configured idle duration,+ falling back to the environment default when unavailable.+ Cached Lua calls cap tracking/hydration/acknowledgement at 50 and cleanup at 100;+ ingestion pipelines at most four scripts within a shard, with one clock lookup+ per ingestion call; single-script groups use a direct call. Record all+ acknowledged siblings after a pipeline error; retry+ only `NOSCRIPT` commands. Retained estimates guard the whole known read window,+ including late arrivals; unknown estimates remain unknown. Locality selection+ still pools 1,000 candidates. Drain the old layout before upgrade+ or shard-count/retention changes. The trace-batch consumer checks existing+ summaries against the job enqueue time so retries skip completed facets while+ later redispatches append updated summaries. Map gauges include shard and+ total series; never sum duplicate reporters or add totals to shard series.`src/features/traceBatching/traceBatchTranscript.ts` measures per-trace assemblyphases, thread count, current-turn/history token estimates and their sum. Eachadmitted trace records full assembled-transcript JSON lengths, renders thediff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex 092a962689..b3cf129b78 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -29,6 +29,11 @@ import * as shared from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";import { env } from "../env";import { TraceBatchMetricsRunner } from "../features/traceBatching/TraceBatchMetricsRunner";+import {+ getTraceBatchFirstKeys,+ getTraceBatchReadinessKeys,+ getTraceBatchShard,+} from "../features/traceBatching/traceBatchReadiness";import {prepareLocalityPartials,selectTraceBatches,@@ -39,6 +44,15 @@ import {import { otelIngestionQueueProcessorBuilder } from "../queues/otelIngestionQueue";import { traceBatchQueueProcessor } from "../queues/traceBatchQueue";import { ClickhouseWriter } from "../services/ClickhouseWriter";+import {+ ensureDefaultTopicFacets,+ listTopicSummaries,+ type TopicsModels,+} from "@langfuse/shared/topics/server";+import {+ summarizeTopicTraceFacets,+ embedTopicSummary,+} from "../features/topics/models";vi.mock("@langfuse/shared/src/server", async (importOriginal) => {const original =@@ -60,6 +74,41 @@ const topicsProjects = vi.hoisted(() => new Set<string>());vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({...(await importOriginal<typeof import("@langfuse/shared/topics/server")>()),isTopicsProjectEnabled: (projectId: string) => topicsProjects.has(projectId),+ getEnabledTopicsModels: async (projectIds: string[]) =>+ new Map(+ projectIds+ .filter((id) => topicsProjects.has(id))+ .map((projectId) => {+ const slot = (+ name: "summary" | "embedding" | "clustering",+ model: string,+ ) => ({+ slot: name,+ provider: "bedrock",+ adapter: "bedrock",+ model,+ connection: { secretKey: "encrypted" },+ });+ return [+ projectId,+ {+ projectId,+ enabled: true,+ summary: slot("summary", "us.openai.gpt-5.6-luna"),+ embedding: {+ ...slot("embedding", "eu.cohere.embed-v4:0"),+ dimensions: 1024,+ },+ clustering: slot("clustering", "us.openai.gpt-5.6-terra"),+ } as TopicsModels,+ ] as const;+ }),+ ),+}));++vi.mock("../features/topics/models", () => ({+ summarizeTopicTraceFacets: vi.fn(),+ embedTopicSummary: vi.fn(),}));vi.mock("../features/evaluation/observationEval", async (importOriginal) => ({@@ -526,8 +575,7 @@ describe("trace batch selection", () => {});describe("trace micro-batch scheduling with Redis", () => {- const dueKey = "{trace-batch}:due";- const stateKey = "{trace-batch}:state";+ const { due: dueKey, state: stateKey } = getTraceBatchReadinessKeys(0);const originalEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;@@ -539,6 +587,8 @@ describe("trace micro-batch scheduling with Redis", () => {const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;+ const originalFirstTtl = env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS;+ const originalShards = env.LANGFUSE_TRACE_BATCH_SHARD_COUNT;const originalMaxTraceEventUpdates =env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES;const originalMaxTraceBytes =@@ -575,6 +625,18 @@ describe("trace micro-batch scheduling with Redis", () => {);}+ async function clearReadiness() {+ for (let shard = 0; shard < env.LANGFUSE_TRACE_BATCH_SHARD_COUNT; shard++) {+ const keys = getTraceBatchReadinessKeys(shard);+ await client().del(+ keys.due,+ keys.state,+ ...getTraceBatchFirstKeys(shard, Date.now()).keys,+ );+ }+ await client().del("{trace-batch}:dispatcher");+ }+beforeEach(async () => {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "true";@@ -586,9 +648,11 @@ describe("trace micro-batch scheduling with Redis", () => {env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 60;env.LANGFUSE_TRACE_BATCH_IDLE_MS = 600_000;env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 7_200_000;+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 86_400_000;+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 1;// The unset default is 2 minutes on DEV; pin the production idle time.env.LANGFUSE_TRACE_BATCH_IDLE_MS = 600_000;- await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");+ await clearReadiness();const redisConnection = createNewRedisInstance();if (!redisConnection) throw new Error("Redis is required for this test");connection = redisConnection;@@ -603,6 +667,7 @@ describe("trace micro-batch scheduling with Redis", () => {runners.length = 0;vi.restoreAllMocks();vi.clearAllMocks();+ await clearReadiness();env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = originalEnabled;env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;@@ -613,16 +678,240 @@ describe("trace micro-batch scheduling with Redis", () => {env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = originalFirstTtl;+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = originalShards;env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES =originalMaxTraceEventUpdates;env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES = originalMaxTraceBytes;await queue.obliterate({ force: true });await queue.close();connection.disconnect();- await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");});afterAll(() => client().disconnect());+ it("re-dispatches late arrivals with the earliest start of the whole trace", async () => {+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 600;+ const start = Date.now() - 20 * 60_000;+ await trackTraceBatchActivity("project", [event("late-trace", start)]);+ const trackedAt = Date.now();+ await vi.waitFor(() =>+ expect(Date.now() - trackedAt).toBeGreaterThanOrEqual(400),+ );+ await makeDue("project", "late-trace");+ await runner().processBatch();+ const [first] = await queue.getJobs(["wait"]);+ expect(first.data.payload.traces[0].minStart).toBe(start);+ expect(await client().zcard(dueKey)).toBe(0);+ const dispatchedAt = Date.now();+ await vi.waitFor(() =>+ expect(Date.now() - dispatchedAt).toBeGreaterThanOrEqual(300),+ );++ const lateStart = start + 10 * 60_000;+ await trackTraceBatchActivity("project", [event("late-trace", lateStart)]);+ await makeDue("project", "late-trace");+ await runner().processBatch();+ const jobs = await queue.getJobs(["wait"]);+ expect(jobs).toHaveLength(2);+ const later = jobs.find((job) => job.id !== first.id)!;+ expect(later.data.payload.traces[0]).toMatchObject({+ minStart: start,+ maxStart: lateStart,+ });+ expect(later.data.payload.traces[0].revision).not.toBe(+ first.data.payload.traces[0].revision,+ );+ const backdatedStart = start + 3 * 60_000;+ await trackTraceBatchActivity("project", [+ event("late-trace", backdatedStart),+ ]);+ await makeDue("project", "late-trace");+ await runner().processBatch();+ const backdated = (await queue.getJobs(["wait"])).find(+ (job) => job.id !== first.id && job.id !== later.id,+ )!;+ expect(backdated.data.payload.traces[0]).toMatchObject({+ minStart: start,+ maxStart: lateStart,+ });+ });++ it("keeps abandoned pending keys through idle time and the recovery grace", async () => {+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 200;+ await trackTraceBatchActivity("project", [event("long-idle")], 500);+ const due = Number(+ await client().zscore(dueKey, member("project", "long-idle")),+ );+ for (const key of [dueKey, stateKey])+ expect(Number(await client().pexpiretime(key))).toBeGreaterThanOrEqual(+ due + 200,+ );+ await vi.waitFor(() =>+ expect(Date.now()).toBeGreaterThanOrEqual(due + 100),+ );+ expect(await client().exists(dueKey, stateKey)).toBe(2);+ await vi.waitFor(async () =>+ expect(await client().exists(dueKey, stateKey)).toBe(0),+ );+ });++ it("expires first-seen history natively while keeping pending state independently", async () => {+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 150;+ const first = Date.now() - 600_000;+ await trackTraceBatchActivity("project", [event("history", first)]);+ const firstKeys = getTraceBatchFirstKeys(0, Date.now()).keys;+ expect(+ (await Promise.all(firstKeys.map((key) => client().hlen(key)))).reduce(+ (sum, count) => sum + count,+ 0,+ ),+ ).toBe(1);+ await makeDue("project", "history");+ await runner().processBatch();+ await vi.waitFor(async () => {+ expect(await client().exists(...firstKeys)).toBe(0);+ });+ const next = first + 300_000;+ await trackTraceBatchActivity("project", [event("history", next)]);+ const state = JSON.parse(+ (await client().hget(stateKey, member("project", "history")))!,+ );+ expect(state).toMatchObject({+ minStart: next,+ eventUpdateCount: 1,+ serializedEventBytes: 100,+ });+ expect(await client().pttl(dueKey)).toBeGreaterThan(150);+ });++ it("distributes skewed projects by trace and dispatches every shard with single-slot scripts", async () => {+ const shardCount = 64;+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = shardCount;+ const evaluate = client().eval.bind(client());+ const slots: string[][] = [];+ vi.spyOn(client(), "eval").mockImplementation(((+ script: string,+ count: number,+ ...args: (string | number | Buffer)[]+ ) => {+ slots.push(args.slice(0, count).map(String));+ return evaluate(script, count, ...args);+ }) as typeof evaluate);+ const evaluateCached = client().evalsha.bind(client());+ vi.spyOn(client(), "evalsha").mockImplementation(((+ scriptHash: string,+ count: number,+ ...args: (string | number | Buffer)[]+ ) => {+ slots.push(args.slice(0, count).map(String));+ return evaluateCached(scriptHash, count, ...args);+ }) as typeof evaluateCached);+ const count = 2_000;+ for (let project = 0; project < 11; project++) {+ const size = project === 0 ? 600 : 140;+ await trackTraceBatchActivity(+ `project-${project}`,+ Array.from({ length: size }, (_, i) => event(`trace-${project}-${i}`)),+ );+ }+ const counts = [];+ for (let shard = 0; shard < shardCount; shard++) {+ const { due } = getTraceBatchReadinessKeys(shard);+ const members = await client().zrange(due, 0, -1);+ counts.push(members.length);+ for (const entry of members)+ expect(getTraceBatchShard(entry)).toBe(shard);+ if (members.length)+ await client().zadd(+ due,+ ...members.flatMap((entry) => [Date.now() - 1_000, entry]),+ );+ }+ expect(counts.reduce((sum, n) => sum + n, 0)).toBe(count);+ expect(Math.max(...counts)).toBeLessThanOrEqual((1.5 * count) / shardCount);+ await runner().processBatch();+ const dispatched = (await queue.getJobs(["wait"])).flatMap(+ (job) => job.data.payload.traces,+ );+ expect(dispatched).toHaveLength(count);+ expect(+ new Set(+ dispatched.map(({ projectId, traceId }) => member(projectId, traceId)),+ ).size,+ ).toBe(count);+ for (const keys of slots) {+ expect(new Set(keys.map((key) => key.match(/\{[^}]+\}/)?.[0])).size).toBe(+ 1,+ );+ }+ });++ it.each(["project", "locality"] as const)(+ "packs one project's traces across readiness shards with %s",+ async (strategy) => {+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 16;+ env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;+ const traceIds = Array.from(+ { length: 60 },+ (_, index) => `cross-shard-${index}`,+ );+ await trackTraceBatchActivity(+ "project",+ traceIds.map((traceId) => event(traceId)),+ );+ for (const traceId of traceIds) {+ const entry = member("project", traceId);+ await client().zadd(+ getTraceBatchReadinessKeys(getTraceBatchShard(entry)).due,+ Date.now() - 1_000,+ entry,+ );+ }+ await runner().processBatch();+ const jobs = await queue.getJobs(["wait"]);+ expect(jobs).toHaveLength(1);+ expect(jobs[0].data.payload.traces).toHaveLength(60);+ for (let shard = 0; shard < 16; shard++) {+ const { due, state } = getTraceBatchReadinessKeys(shard);+ expect(await client().zcard(due)).toBe(0);+ expect(await client().hlen(state)).toBe(0);+ }+ },+ );++ it("acknowledges enqueued batches by shard across a hydration window", async () => {+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 16;+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 30;+ const traceIds = Array.from(+ { length: 120 },+ (_, index) => `trace-${index}`,+ );+ await trackTraceBatchActivity(+ "project",+ traceIds.map((traceId) => event(traceId)),+ );+ for (const traceId of traceIds) {+ const entry = member("project", traceId);+ await client().zadd(+ getTraceBatchReadinessKeys(getTraceBatchShard(entry)).due,+ Date.now() - 1_000,+ entry,+ );+ }+ const evaluate = vi.spyOn(client(), "evalsha");+ await runner().processBatch();+ expect(await queue.getWaitingCount()).toBe(4);+ const acknowledgements = evaluate.mock.calls.filter(+ ([, keyCount]) => Number(keyCount) > 2,+ );+ expect(acknowledgements.length).toBeLessThanOrEqual(16);+ for (let shard = 0; shard < 16; shard++) {+ const { due, state } = getTraceBatchReadinessKeys(shard);+ expect(await client().zcard(due)).toBe(0);+ expect(await client().hlen(state)).toBe(0);+ }+ });+it("samples prefixed map memory and paused/delayed queue work, then reports drained zeros", async () => {const now = Date.now();const clock = vi.spyOn(Date, "now").mockReturnValue(now);@@ -645,11 +934,11 @@ describe("trace micro-batch scheduling with Redis", () => {const evaluate = vi.spyOn(client(), "eval");await metrics["execute"]();- for (const key of ["due", "state"]) {+ for (const key of ["due", "state", "first"]) {expect(recordGauge).toHaveBeenCalledWith("langfuse.trace_batch.redis_key_entries",1,- { key, unit: "records" },+ { key, shard: "total", unit: "records" },);const sample = vi.mocked(recordGauge)@@ -679,7 +968,7 @@ describe("trace micro-batch scheduling with Redis", () => {clock.mockReturnValue(now + 30_000);await metrics["execute"]();expect(evaluate).toHaveBeenCalledTimes(1);- await client().del(dueKey, stateKey);+ await clearReadiness();await queue.drain(true);vi.mocked(recordGauge).mockClear();clock.mockReturnValue(now + 60_000);@@ -689,7 +978,7 @@ describe("trace micro-batch scheduling with Redis", () => {"langfuse.trace_batch.active_reads",0,);- expect(vi.mocked(recordGauge).mock.calls).toHaveLength(7);+ expect(vi.mocked(recordGauge).mock.calls).toHaveLength(15);expect(vi.mocked(recordGauge).mock.calls.every(([, value]) => value === 0),).toBe(true);@@ -820,6 +1109,63 @@ describe("trace micro-batch scheduling with Redis", () => {expect.objectContaining({ projectId: "project", traceId, reason }),);}+ await trackTraceBatchActivity("project", [+ event("many-updates", 1_000_004),+ event("large-input", 1_000_001),+ ]);+ for (const [traceId, eventUpdateCount, serializedEventBytes] of [+ ["many-updates", 5, 500],+ ["large-input", 2, 5_100],+ ] as const)+ expect(+ JSON.parse(+ (await client().hget(stateKey, member("project", traceId)))!,+ ),+ ).toMatchObject({ eventUpdateCount, serializedEventBytes });+ await makeDue("project", "many-updates", "large-input");+ await runner().processBatch();+ expect(await queue.getWaitingCount()).toBe(1);+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ });++ it("uses retained full-trace estimates for batch budgets after successful dispatch", async () => {+ const original = env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES;+ env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES = 6;+ try {+ const ingest = async (start: number) => {+ await trackTraceBatchActivity("project", [+ event("a", start),+ event("a", start),+ event("b", start),+ event("b", start),+ ]);+ await makeDue("project", "a", "b");+ await runner().processBatch();+ };+ await ingest(1_000_000);+ const [first] = await queue.getJobs(["wait"]);+ expect(first.data.payload.traces).toHaveLength(2);+ await ingest(1_000_001);+ const lateJobs = (await queue.getJobs(["wait"])).filter(+ (job) => job.id !== first.id,+ );+ expect(lateJobs).toHaveLength(2);+ expect(lateJobs.flatMap((job) => job.data.payload.traces)).toHaveLength(+ 2,+ );+ for (const job of lateJobs) {+ expect(job.data.payload.traces).toHaveLength(1);+ expect(job.data.payload.traces[0].minStart).toBe(1_000_000);+ }+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.estimated_event_update_count",+ 4,+ { scope: "trace", strategy: "project" },+ );+ } finally {+ env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES = original;+ }});it("does not report an oversized trace as excluded when new activity keeps it pending", async () => {@@ -832,13 +1178,18 @@ describe("trace micro-batch scheduling with Redis", () => {),);await makeDue("project", "reactivated");- const evaluate = client().eval.bind(client());+ const evaluate = client().evalsha.bind(client());let reactivated = false;- vi.spyOn(client(), "eval").mockImplementation((async (- script: string,+ vi.spyOn(client(), "evalsha").mockImplementation((async (+ scriptHash: string,+ keyCount: number,...args: (string | number)[]) => {- if (!reactivated && script.includes("cjson.decode(state).revision ==")) {+ if (+ !reactivated &&+ keyCount > 2 &&+ Number(args[keyCount]) === env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS+ ) {reactivated = true;const traceMember = member("project", "reactivated");const state = JSON.parse((await client().hget(stateKey, traceMember))!);@@ -848,7 +1199,7 @@ describe("trace micro-batch scheduling with Redis", () => {JSON.stringify({ ...state, revision: randomUUID() }),);}- return evaluate(script, ...args);+ return evaluate(scriptHash, keyCount, ...args);}) as typeof evaluate);await runner().processBatch();@@ -1066,6 +1417,42 @@ describe("trace micro-batch scheduling with Redis", () => {});it("ingests through BullMQ, postpones readiness on new activity, and consumes full ClickHouse payloads once due", async () => {+ const orgId = randomUUID();+ const projectId = randomUUID();+ await prisma.organization.create({+ data: {+ id: orgId,+ name: "Trace batch late arrivals",+ projects: { create: { id: projectId, name: "Trace batch" } },+ },+ });+ const usage = {+ providedUsageDetails: {},+ usageDetails: {},+ providedCostDetails: {},+ costDetails: {},+ };+ vi.mocked(summarizeTopicTraceFacets).mockImplementation(+ async (_models, facets) => ({+ output: Object.fromEntries(+ facets.map(({ key }) => [+ key,+ {+ notes: "The complete trace was read.",+ summary: `summary-${vi.mocked(summarizeTopicTraceFacets).mock.calls.length}`,+ status: "applicable" as const,+ },+ ]),+ ),+ ...usage,+ }),+ );+ vi.mocked(embedTopicSummary).mockImplementation(+ async (_model, _text, dimensions) => ({+ embedding: Array.from({ length: dimensions }, () => 0.1),+ ...usage,+ }),+ );const original = {LANGFUSE_TRACE_BATCH_IDLE_MS: env.LANGFUSE_TRACE_BATCH_IDLE_MS,LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,@@ -1110,21 +1497,25 @@ describe("trace micro-batch scheduling with Redis", () => {batchWorker.waitUntilReady(),batchEvents.waitUntilReady(),]);- const projectId = randomUUID();// The evaluator sampler assigns these scores 0.0593, 0.0291 and 0.7116.const traceId = "00000000000000000000000000000001";const otherTraceId = "00000000000000000000000000000003";const excludedTraceId = "00000000000000000000000000000002";- const input = JSON.stringify({ message: "input".repeat(1_000) });- const output = JSON.stringify({ message: "output".repeat(1_000) });+ const input = JSON.stringify([+ { role: "user", content: "input".repeat(1_000) },+ ]);+ const output = JSON.stringify({+ role: "assistant",+ content: "output".repeat(1_000),+ });const metadata = JSON.stringify({ context: "metadata".repeat(1_000) });- const nano = BigInt(Date.now()) * 1_000_000n;- const timestamp = {- low: Number(nano & 0xffffffffn),- high: Number(nano >> 32n),- unsigned: true,- };- async function ingest(traceIds: string[]) {+ const nano = BigInt(Date.now() - 20 * 60_000) * 1_000_000n;+ async function ingest(traceIds: string[], start = nano) {+ const timestamp = {+ low: Number(start & 0xffffffffn),+ high: Number(start >> 32n),+ unsigned: true,+ };// S3 transport and unrelated evaluator configuration are stubbed;// OTLP conversion, writer, Redis scripts and both queue workers are real.vi.mocked(getS3EventStorageClient).mockReturnValue({@@ -1144,7 +1535,7 @@ describe("trace micro-batch scheduling with Redis", () => {endTimeUnixNano: timestamp,status: { code: 2, message: "tool call failed" },attributes: Object.entries({- "langfuse.observation.type": "span",+ "langfuse.observation.type": "generation","langfuse.observation.input": input,"langfuse.observation.output": output,"langfuse.observation.metadata": metadata,@@ -1234,6 +1625,8 @@ describe("trace micro-batch scheduling with Redis", () => {excludedObservations++;}expect(excludedObservations).toBe(2);+ topicsProjects.add(projectId);+ const facets = await ensureDefaultTopicFacets(projectId);env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0;await waitUntilRedisTime(renewedDue);@@ -1264,6 +1657,79 @@ describe("trace micro-batch scheduling with Redis", () => {expect(await client().hlen(stateKey)).toBe(0);await dispatcher.processBatch();expect(await queue.getCompletedCount()).toBe(1);+ expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(2);+ const summaries = async () =>+ (+ await Promise.all(+ facets.map((facet) =>+ listTopicSummaries(+ projectId,+ {+ traceIds: [traceId, otherTraceId],+ facetId: facet.id,+ facetVersion: facet.versions[0].version,+ },+ {+ from: new Date(Number(nano / 1_000_000n) - 1),+ to: new Date(),+ },+ ),+ ),+ )+ ).flat();+ const firstSummaries = await summaries();+ expect(firstSummaries).toHaveLength(8);++ // One late event stays within the original buffer, while the other is+ // ten minutes away and requires the retained earliest start.+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;+ await ingest([traceId], nano + 60_000n * 1_000_000n);+ await ingest([otherTraceId], nano + 10n * 60_000n * 1_000_000n);+ await writer.flushAll(true);+ await makeDue(projectId, traceId, otherTraceId);+ await dispatcher.processBatch();+ const lateBatch = (+ await queue.getJobs(["wait", "active", "completed"])+ ).find((job) => job.id !== batch.id)!;+ for (const trace of lateBatch.data.payload.traces)+ expect(trace.minStart).toBe(Number(nano / 1_000_000n));+ const lateResult = await lateBatch.waitUntilFinished(batchEvents, 10_000);+ expect(lateResult).toMatchObject({ observationCount: 5, traceCount: 2 });+ expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(4);++ await ingest([otherTraceId], nano + 3n * 60_000n * 1_000_000n);+ await writer.flushAll(true);+ await makeDue(projectId, otherTraceId);+ await dispatcher.processBatch();+ const backdatedBatch = (+ await queue.getJobs(["wait", "active", "completed"])+ ).find((job) => job.id !== batch.id && job.id !== lateBatch.id)!;+ expect(backdatedBatch.data.payload.traces).toMatchObject([+ {+ minStart: Number(nano / 1_000_000n),+ maxStart: Number(nano / 1_000_000n) + 10 * 60_000,+ },+ ]);+ expect(+ await backdatedBatch.waitUntilFinished(batchEvents, 10_000),+ ).toMatchObject({ observationCount: 3, traceCount: 1 });+ expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(5);+ const latest = await summaries();+ expect(latest).toHaveLength(8);+ for (const summary of latest) {+ expect(summary.unitStartTime).toBe(+ firstSummaries.find((row) => row.traceId === summary.traceId)!+ .unitStartTime,+ );+ expect(summary.summary).not.toBe(+ firstSummaries.find(+ (row) =>+ row.traceId === summary.traceId &&+ row.facetId === summary.facetId,+ )!.summary,+ );+ }} finally {try {await Promise.all([ingestionWorker.close(), batchWorker.close()]);@@ -1272,6 +1738,8 @@ describe("trace micro-batch scheduling with Redis", () => {await ingestionQueue.close();await writer.shutdown();} finally {+ topicsProjects.delete(projectId);+ await prisma.organization.deleteMany({ where: { id: orgId } });Object.assign(env, original);}}@@ -1295,6 +1763,16 @@ describe("trace micro-batch scheduling with Redis", () => {}});+ it("does not load idle settings when every trace is sampled out", async () => {+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = Number.MIN_VALUE;+ vi.mocked(shared.getTracingReadinessSettings).mockClear();++ await trackTraceBatchActivity("project", [event("sampled-out")]);++ expect(shared.getTracingReadinessSettings).not.toHaveBeenCalled();+ expect(await client().hlen(stateKey)).toBe(0);+ });+it("keeps stable nested trace samples across replay and counts decisions once per trace in each ingestion batch", async () => {const traceIds = Array.from({ length: 1_000 }, (_, i) =>i.toString(16).padStart(32, "0"),@@ -1343,26 +1821,48 @@ describe("trace micro-batch scheduling with Redis", () => {["eligible", 1_000],...(expectedCount > 0 ? [["recorded", expectedCount] as const] : []),] as const) {- expect(recordIncrement).toHaveBeenCalledWith(- "langfuse.trace_batch.event_updates",- count * 3,- { stage },- );- expect(recordIncrement).toHaveBeenCalledWith(- "langfuse.trace_batch.serialized_event_bytes",- count * 350,- { stage },- );+ for (const [metric, total] of [+ ["event_updates", count * 3],+ ["serialized_event_bytes", count * 350],+ ] as const)+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(+ ([name, , tags]) =>+ name === `langfuse.trace_batch.${metric}` &&+ tags?.stage === stage,+ )+ .reduce((sum, [, value]) => sum + value, 0),+ ).toBe(total);}previous = selected;}});it("counts only acknowledged update volume when tracking fails after a chunk", async () => {- const evaluate = client().eval.bind(client());- vi.spyOn(client(), "eval")- .mockImplementationOnce(evaluate)- .mockRejectedValueOnce(new Error("tracking connection failed"));+ await trackTraceBatchActivity("warm-up", [event("trace")]);+ await clearReadiness();+ vi.mocked(recordIncrement).mockClear();+ const fallback = vi.spyOn(client(), "eval");+ const pipeline = client().pipeline.bind(client());+ let commands = 0;+ vi.spyOn(client(), "pipeline").mockImplementation(() => {+ const batch = pipeline();+ const evaluate = batch.evalsha.bind(batch);+ vi.spyOn(batch, "evalsha").mockImplementation(((+ scriptHash: string,+ keyCount: number,+ ...args: (string | number | Buffer)[]+ ) => {+ // Fail one command before mutation; the other pipelined chunks still execute.+ if (++commands === 2) args[1] = args[0];+ // Simulate a cache miss for one sibling without flushing shared Redis scripts.+ if (commands === 3) scriptHash = "0".repeat(40);+ return evaluate(scriptHash, keyCount, ...args);+ }) as typeof evaluate);+ return batch;+ });await expect(trackTraceBatchActivity("project", [@@ -1373,7 +1873,15 @@ describe("trace micro-batch scheduling with Redis", () => {]),).resolves.toBeUndefined();- expect(await client().hlen(stateKey)).toBe(1_000);+ const recorded = await client().hlen(stateKey);+ expect(recorded).toBe(150);+ expect(commands).toBe(4);+ expect(fallback).toHaveBeenCalledOnce();+ for (const state of await client().hvals(stateKey))+ expect(JSON.parse(state)).toMatchObject({+ eventUpdateCount: 1,+ serializedEventBytes: 7,+ });expect(recordIncrement).toHaveBeenCalledWith("langfuse.trace_batch.event_updates",1_001,@@ -1384,16 +1892,20 @@ describe("trace micro-batch scheduling with Redis", () => {expect.anything(),{ stage: "sampled" },);- expect(recordIncrement).toHaveBeenCalledWith(- "langfuse.trace_batch.event_updates",- 1_000,- { stage: "recorded" },- );- expect(recordIncrement).toHaveBeenCalledWith(- "langfuse.trace_batch.serialized_event_bytes",- 7_000,- { stage: "recorded" },- );+ for (const [metric, count] of [+ ["event_updates", recorded],+ ["serialized_event_bytes", recorded * 7],+ ] as const)+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(+ ([name, , tags]) =>+ name === `langfuse.trace_batch.${metric}` &&+ tags?.stage === "recorded",+ )+ .reduce((sum, [, value]) => sum + value, 0),+ ).toBe(count);expect(recordIncrement).not.toHaveBeenCalledWith("langfuse.trace_batch.event_updates",1,@@ -1593,7 +2105,7 @@ describe("trace micro-batch scheduling with Redis", () => {},);- it("keeps legacy estimates unknown through updates and restores them after dispatch", async () => {+ it("keeps unknown estimates through updates and late-arrival dispatch", async () => {const legacyStates = [{},{ eventUpdateCount: 4 },@@ -1611,6 +2123,11 @@ describe("trace micro-batch scheduling with Redis", () => {...estimates,}),);+ await client().zadd(+ dueKey,+ Date.now() + 600_000,+ member("project", legacyIds[index]),+ );}await trackTraceBatchActivity("project", [...legacyIds.map((traceId) => event(traceId, 1_000_001, 20)),@@ -1667,34 +2184,26 @@ describe("trace micro-batch scheduling with Redis", () => {legacyIds.map((traceId) => event(traceId, 1_000_002, 20)),);for (const traceId of legacyIds) {- expect(- JSON.parse(- (await client().hget(stateKey, member("project", traceId)))!,- ),- ).toMatchObject({- eventUpdateCount: 1,- serializedEventBytes: 20,- });+ const state = JSON.parse(+ (await client().hget(stateKey, member("project", traceId)))!,+ );+ expect(state.eventUpdateCount).toBeUndefined();+ expect(state.serializedEventBytes).toBeUndefined();}await makeDue("project", ...legacyIds);await dispatcher.processBatch();- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.estimated_event_update_count",- 3,- { scope: "batch", strategy: "project" },- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.estimated_serialized_event_bytes",- 60,- { scope: "batch", strategy: "project" },- );expect(vi- .mocked(recordIncrement)- .mock.calls.some(- ([name]) => name === "langfuse.trace_batch.estimates_unavailable",+ .mocked(recordDistribution)+ .mock.calls.filter(([name]) =>+ name.startsWith("langfuse.trace_batch.estimated_"),),- ).toBe(false);+ ).toHaveLength(0);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.estimates_unavailable",+ 1,+ { scope: "batch", strategy: "project" },+ );});it.each(["project", "locality"] as const)(@@ -1862,15 +2371,11 @@ describe("trace micro-batch scheduling with Redis", () => {const hydratedRevision = JSON.parse((await client().hget(stateKey, targetMember))!,).revision;- const evaluate = client().eval.bind(client());+ const evaluate = client().evalsha.bind(client());let reactivated = false;- vi.spyOn(client(), "eval").mockImplementation(async (...args) => {+ vi.spyOn(client(), "evalsha").mockImplementation(async (...args) => {const result = await evaluate(...args);- if (- !reactivated &&- typeof args[0] === "string" &&- args[0].includes("local result = {}")- ) {+ if (!reactivated && Number(args[1]) === 2 && args.length > 6) {reactivated = true;await trackTraceBatchActivity("project", [event(early[0].traceId, 11 * minute),@@ -1980,12 +2485,12 @@ describe("trace micro-batch scheduling with Redis", () => {});it("expires abandoned pending keys natively without another ingestion or dispatcher run", async () => {- env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 100;env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 300;await trackTraceBatchActivity("project", [event("abandoned")]);for (const key of [dueKey, stateKey]) {expect(await client().pttl(key)).toBeGreaterThan(0);- expect(await client().pttl(key)).toBeLessThanOrEqual(300);+ expect(await client().pttl(key)).toBeLessThanOrEqual(400);}await vi.waitFor(async () => {expect(await client().exists(dueKey, stateKey)).toBe(0);@@ -1999,7 +2504,7 @@ describe("trace micro-batch scheduling with Redis", () => {await trackTraceBatchActivity("project", [event("long-idle")]);const initialDeadline = Number(await client().pexpiretime(dueKey));await vi.waitFor(() =>- expect(Date.now()).toBeGreaterThan(initialDeadline - 60_000 + 10),+ expect(Date.now()).toBeGreaterThan(initialDeadline - 70_000 + 10),);env.LANGFUSE_TRACE_BATCH_IDLE_MS = 20_000;await trackTraceBatchActivity("project", [event("later")]);@@ -2018,6 +2523,24 @@ describe("trace micro-batch scheduling with Redis", () => {);expect(await client().zcard(dueKey)).toBe(3);expect(await client().hlen(stateKey)).toBe(3);++ const roundedDeadline =+ Math.ceil((Date.now() + 4 * 60 * 60_000) / 100_000_000) * 100_000_000;+ for (const key of [dueKey, stateKey])+ await client().pexpireat(key, roundedDeadline);+ vi.mocked(recordIncrement).mockClear();+ await trackTraceBatchActivity("project", [event("rounded-expiry")]);+ expect(recordIncrement).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.tracking_errors",+ expect.anything(),+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.event_updates",+ 1,+ { stage: "recorded" },+ );+ for (const key of [dueKey, stateKey])+ expect(Number(await client().pexpiretime(key))).toBe(roundedDeadline);});it("drains and removes queued jobs without queries when reads are disabled", async () => {@@ -2096,8 +2619,10 @@ describe("trace micro-batch scheduling with Redis", () => {);await makeDue("project", "recent");await trackTraceBatchActivity("project", [event("refreshed", 2_000_000)]);- expect(await client().zcard(dueKey)).toBe(3);- expect(await client().hlen(stateKey)).toBe(3);+ const pending = await client().zcard(dueKey);+ expect(pending).toBeGreaterThan(2);+ expect(pending).toBeLessThan(expired.length + 2);+ expect(await client().hlen(stateKey)).toBe(pending);expect(await client().hget(stateKey, member("project", "recent")),).not.toBeNull();@@ -2106,20 +2631,30 @@ describe("trace micro-batch scheduling with Redis", () => {).toBeGreaterThan(Date.now());expect(recordIncrement).toHaveBeenCalledWith("langfuse.trace_batch.expired_traces",- 1_000,+ expired.length + 2 - pending,{ source: "ingestion" },);- await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);+ await vi.waitFor(async () => {+ await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);+ expect(await client().zcard(dueKey)).toBe(2);+ });expect((await client().hkeys(stateKey)).sort()).toEqual([member("project", "recent"),member("project", "refreshed"),]);expect(await client().zcard(dueKey)).toBe(2);- expect(recordIncrement).toHaveBeenCalledWith(- "langfuse.trace_batch.expired_traces",- 1,- { source: "ingestion" },+ const expirySamples = vi+ .mocked(recordIncrement)+ .mock.calls.filter(+ ([name, , tags]) =>+ name === "langfuse.trace_batch.expired_traces" &&+ tags?.source === "ingestion",+ )+ .map(([, count]) => count);+ expect(expirySamples.reduce((sum, count) => sum + count, 0)).toBe(+ expired.length,);+ expect(expirySamples.every((count) => count < expired.length)).toBe(true);expect(await queue.getWaitingCount()).toBe(0);});@@ -2190,6 +2725,25 @@ describe("trace micro-batch scheduling with Redis", () => {},);+ it("acknowledges successful jobs when a later enqueue fails", async () => {+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 1;+ await trackTraceBatchActivity("project", [event("first"), event("second")]);+ await makeDue("project", "first", "second");+ const add = queue.add.bind(queue);+ vi.spyOn(queue, "add")+ .mockImplementationOnce((...args) => add(...args))+ .mockRejectedValueOnce(new Error("enqueue unavailable"));+ const dispatcher = runner();+ await dispatcher.processBatch();+ expect(await queue.getWaitingCount()).toBe(1);+ expect(await client().zcard(dueKey)).toBe(1);+ expect(await client().hlen(stateKey)).toBe(1);+ await dispatcher.processBatch();+ expect(await queue.getWaitingCount()).toBe(2);+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ });+it.each(["updated", "deleted", "expired"] as const)("preserves arrivals during enqueue, including deleted/recreated state: %s",async (mode) => {@@ -2222,6 +2776,10 @@ describe("trace micro-batch scheduling with Redis", () => {const [job] = await queue.getJobs(["wait"]);expect(current.revision).not.toBe(job.data.payload.traces[0].revision);expect(current.maxStart).toBe(2_000_000);+ expect(current).toMatchObject({+ eventUpdateCount: 2,+ serializedEventBytes: 200,+ });await makeDue("project", "trace");await dispatcher.processBatch();expect(await queue.getWaitingCount()).toBe(2);@@ -2238,7 +2796,7 @@ describe("trace micro-batch scheduling with Redis", () => {vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {const job = await add(...args);// The next Redis operation is the post-enqueue acknowledgement.- vi.spyOn(client(), "eval").mockRejectedValueOnce(+ vi.spyOn(client(), "evalsha").mockRejectedValueOnce(new Error("ACK unavailable"),);return job;diff --git a/worker/src/env.ts b/worker/src/env.tsindex ae53a57ae3..3e0fef80fb 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -212,6 +212,14 @@ const EnvSchema = z.object({.enum(["project", "locality"]).default("project"),LANGFUSE_TRACE_BATCH_IDLE_MS: z.coerce.number().int().positive(),+ LANGFUSE_TRACE_BATCH_SHARD_COUNT: z.coerce.number().int().min(1).max(256),+ LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS: z.coerce+ .number()+ .int()+ .positive()+ .max(2 * 24 * 60 * 60 * 1_000)+ .default(24 * 60 * 60 * 1_000),+ // Recovery grace after readiness; whole-key expiry includes the idle interval.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS: z.coerce.number().int()@@ -890,6 +898,10 @@ const parseEnv = (): ParsedEnv => {source.LANGFUSE_TRACE_BATCH_IDLE_MS =source.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV" ? "120000" : "600000";}+ if (source.LANGFUSE_TRACE_BATCH_SHARD_COUNT === undefined) {+ source.LANGFUSE_TRACE_BATCH_SHARD_COUNT =+ source.REDIS_CLUSTER_ENABLED === "true" ? "16" : "1";+ }const parsed = EnvSchema.parse(source);validateV4Flags(parsed);validateInAppAgentSandboxConfig(parsed);diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.tsindex 9eb6adaa5e..40c6f5804e 100644--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts@@ -133,11 +133,12 @@ beforeEach(() => {});describe("summarizeAssembledTrace", () => {- it("summarizes the assembled transcript once per current facet and skips a finished retry", async () => {+ it("skips a finished job retry but summarizes a later redispatch", async () => {await summarizeAssembledTrace({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -183,18 +184,16 @@ describe("summarizeAssembledTrace", () => {expect.any(Number),{ stage: "trace", outcome: "success", unit: "milliseconds" },);- state.stored.mockImplementation(async (_projectId, _filter, timeRange) =>- new Date(written.unitStartTime) >= timeRange.from &&- new Date(written.unitStartTime) < timeRange.to- ? [written]- : [],- );+ state.stored.mockResolvedValue([+ { ...written, processedAt: "2026-09-22T12:02:00.000Z" },+ ]);state.summarize.mockClear();state.write.mockClear();await summarizeAssembledTrace({projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -206,7 +205,8 @@ describe("summarizeAssembledTrace", () => {await summarizeAssembledTrace({projectId: "project-a",traceId: "trace-1",- traceTimestamp: "2026-09-22T11:59:00.000Z",+ traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:03:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -214,9 +214,11 @@ describe("summarizeAssembledTrace", () => {facets: state.facets(),});expect(state.summarize).toHaveBeenCalledOnce();+ expect(state.write).toHaveBeenCalledOnce();expect(state.write.mock.calls[0][0][0].unitStartTime).toBe(- "2026-09-22T11:59:00.000Z",+ written.unitStartTime,);+ expect(state.stored).toHaveBeenCalledTimes(3);});it("summarizes all pending facets in one call and records its usage once", async () => {@@ -238,6 +240,7 @@ describe("summarizeAssembledTrace", () => {projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -266,6 +269,7 @@ describe("summarizeAssembledTrace", () => {projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -298,6 +302,7 @@ describe("summarizeAssembledTrace", () => {projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -355,6 +360,7 @@ describe("summarizeAssembledTrace", () => {projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -391,6 +397,7 @@ describe("summarizeAssembledTrace", () => {projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -433,6 +440,7 @@ describe("summarizeAssembledTrace", () => {projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,@@ -470,6 +478,7 @@ describe("summarizeAssembledTrace", () => {projectId: "project-a",traceId: "trace-1",traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),environment: "default",traceName: "agent-turn",transcript,diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.tsindex ca63f351c6..79b7646957 100644--- a/worker/src/features/topics/summarizeAssembledTrace.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.ts@@ -40,6 +40,7 @@ type AssembledTraceInput = {projectId: string;traceId: string;traceTimestamp: string;+ jobEnqueuedAt: Date;environment: string;traceName: string;transcript: Transcript | null;@@ -49,7 +50,7 @@ type AssembledTraceInput = {/*** Summarizes one trace from the transcript the batch job already assembled,- * with one model call for all of its pending facets. Does not load+ * with one model call for all of its configured facets. Does not load* observations again.*/export async function summarizeAssembledTrace(@@ -86,11 +87,6 @@ async function summarizeEnabledTrace(const config = topicProcessingConfigSchema.parse({summaryModel: models.summary.model,});- const timestamp = Date.parse(input.traceTimestamp);- const timeRange = {- from: new Date(timestamp),- to: new Date(timestamp + 1),- };const pending: {name: string;version: TopicFacetVersion;@@ -101,6 +97,11 @@ async function summarizeEnabledTrace(...facets.filter((facet) => facet.isBuiltIn),...facets.filter((facet) => !facet.isBuiltIn),];+ const timestamp = Date.parse(input.traceTimestamp);+ const timeRange = {+ from: new Date(timestamp),+ to: new Date(timestamp + 1),+ };for (const facet of ordered) {const version = facet.versions[0];if (facet.projectId !== input.projectId || !version) continue;@@ -113,8 +114,14 @@ async function summarizeEnabledTrace(},timeRange,);- if (!stored.length)- pending.push({ name: facet.name, version, builtIn: facet.isBuiltIn });+ if (+ stored.some(+ (summary) =>+ Date.parse(summary.processedAt) >= input.jobEnqueuedAt.getTime(),+ )+ )+ continue;+ pending.push({ name: facet.name, version, builtIn: facet.isBuiltIn });}if (!pending.length) return "unchanged";diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex cb56fef35f..95662d8a69 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -7,31 +7,54 @@ flags and transcript-metrics switch below default to `false`; an ordinary releasneeds no infrastructure changes. The four enablement flags stay commented in envtemplates. The local dev template sets the idle time below.+## Readiness requirements++- Every admitted event refreshes the trace's idle deadline. Tracking uses the+ project's configured idle interval, or the deployment default when unset.+- Keep pending scheduling state separate from retained historical bounds. Enqueue+ acknowledgement clears pending state; history survives to support later arrivals.+- A late arrival within history retention opens a new pending window and reads the+ whole known trace, using the earliest and latest retained event starts plus the+ existing two-minute buffer. Backdated arrivals must preserve the previous tail.+- The consumer checks existing summaries against the job enqueue time. Retries+ skip facets completed by this job; later redispatches append updated summaries.+- Bounds, counters, revision and due time are merged atomically. Readiness is+ revalidated before enqueue, and acknowledgement must preserve any newer revision.+- Pending state must survive its idle interval plus recovery grace. Historical+ values have an independent logical TTL and expire without field scans.+- Use Redis 7-compatible operations on standalone Redis, Sentinel and Cluster.+ Every multi-key Lua call stays in one slot; cleanup and Lua batches are bounded.+ One renewable dispatcher lease coordinates scheduling across all shards.+- Redis loss and events outside retained history can leave gaps. There is no+ reconciler, and retained timestamps cannot recover data that was never observed.+## Controls-| Variable | Default | Role |-| --- | --- | --- |-| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |-| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |-| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |-| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |-| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |-| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `0` | Fraction of other projects admitted by ingestion, from 0 to 1. Unset or `0` is off. Topics-enabled projects are always admitted. |-| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |-| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |-| `LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES` | `1500` | Estimated event updates per multi-trace job; `0` disables. |-| `LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES` | `67108864` | Estimated serialized event bytes per multi-trace job (64 MiB); `0` disables. |-| `LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` | `5000` | Exclude a trace whose pending window exceeds this many event updates; `0` disables. |-| `LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` | `52428800` | Exclude a trace whose pending window exceeds these serialized bytes (50 MiB); `0` disables. |-| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |-| `LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS` | `40000` | ClickHouse client socket idle timeout in milliseconds, including pauses for summary backpressure. Independent of the 30-second server query execution limit. |-| `LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES` | `104857600` | Queued input/output/metadata payload threshold (100 MiB). Includes the active summary; excludes the current trace, tool fields, transport buffers and JavaScript object overhead. |-| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |-| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |-| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Default idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. A project's `tracing_readiness_settings.trace_idle_ms` (60 s to 60 min) overrides it, capped at the pending TTL minus the dispatch interval. The project value is cached in Redis for 5 minutes; saving replaces it, so the next activity reads the new value. A failed read uses the default and is not cached. |-| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |-| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |+| Variable | Default | Role |+| ------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |+| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `0` | Fraction of other projects admitted by ingestion, from 0 to 1. Unset or `0` is off. Topics-enabled projects are always admitted. |+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |+| `LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES` | `1500` | Estimated event updates per multi-trace job; `0` disables. |+| `LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES` | `67108864` | Estimated serialized event bytes per multi-trace job (64 MiB); `0` disables. |+| `LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` | `5000` | Exclude a trace whose retained read-window estimate exceeds this many event updates; `0` disables. |+| `LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` | `52428800` | Exclude a trace whose retained read-window estimate exceeds these serialized bytes (50 MiB); `0` disables. |+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |+| `LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS` | `40000` | ClickHouse client socket idle timeout in milliseconds, including pauses for summary backpressure. Independent of the 30-second server query execution limit. |+| `LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES` | `104857600` | Queued input/output/metadata payload threshold (100 MiB). Includes the active summary; excludes the current trace, tool fields, transport buffers and JavaScript object overhead. |+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Default idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. A project's `tracing_readiness_settings.trace_idle_ms` (60 seconds to 60 minutes) overrides it, capped at the pending TTL minus the dispatch interval. The project value is cached in Redis for 5 minutes; saving replaces it, and a failed read uses the default without caching. |+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Recovery grace after a trace becomes due (2 hours). Pending keys expire no earlier than the latest required due-plus-grace deadline. |+| `LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS` | `86400000` | Historical event-start bounds and size estimates retained after activity or dispatch (24 hours; at most 48 hours). Internal and cloud-gated. |+| `LANGFUSE_TRACE_BATCH_SHARD_COUNT` | `auto` | 16 on Redis Cluster; 1 on standalone/Sentinel. Override with 1–256. Fixed across Redis resizing; drain before changing. |+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |Producer, dispatcher and consumer can run independently. Sampling reuses theevaluator's deterministic trace-ID sampler; every observation of the same trace@@ -41,42 +64,173 @@ processing, and the shared queue accessor, including admin queue inspection.## Redis state and expiry-- `{trace-batch}:due`: sorted set, with JSON `[projectId, traceId]` members and+- `{trace-batch:<shard>}:due`: sorted set, with JSON `[projectId, traceId]` members andRedis server time plus idle delay as the score.-- `{trace-batch}:state`: hash with the same members and JSON- `{minStart, maxStart, revision, eventUpdateCount, serializedEventBytes}` values. Bounds describe observed event start+- `{trace-batch:<shard>}:state`: hash with the same members and JSON+ `{minStart, maxStart, revision, historyBucket, eventUpdateCount, serializedEventBytes}` values. Bounds describe observed event starttimes; revision protects newer ingestion from an older dispatch acknowledgment.- `{trace-batch}:dispatcher`: renewable dispatcher lease.+- `{trace-batch:<shard>}:first:<bucket>`: hourly hashes retaining the minimum and+ maximum event starts independently of pending state. Fields are the first 16 binary+ bytes of SHA-256 of the JSON project/trace pair; values are MessagePack+ `[minStart, maxStart, expiresAt, eventUpdateCount, serializedEventBytes]`.+ A bounds-only tuple has unknown estimates. This accepts the negligible collision probability of a+ 128-bit identity. Key shape is independent of the processing unit.++Sharding uses the shared SHA-256 shard helper on the complete project/trace pair,+so a large project spreads across shards. Every Lua call uses only keys with+one shard's hash tag. The single dispatcher lease covers all shards; batch+packing combines ready traces from every shard. Queue slots are unchanged.++When pending state is absent or has no bucket pointer, tracking looks up the+bounded set of live time buckets (26 keys for the default TTL), never scans their+fields, and atomically merges the historical bounds. Valid pending state contains+both bounds, cumulative size estimates and its current history bucket, allowing direct movement to the new+bucket on subsequent activity and guarded acknowledgement. The Redis server clock owns+both retention and readiness. Values with an elapsed deadline are ignored;+whole-key expiry at bucket end plus retention removes storage without scans.+Physical removal can lag logical expiry by up to one hour (or the TTL for a+shorter test override). Activity and guarded acknowledgement refresh retention,+so the post-dispatch window lasts the configured TTL independently of idle delay.+One live copy per identity+is retained under a consistent TTL.No event payloads are stored here. Atomic Lua updates accumulate estimates and-refresh bounds, revision and readiness. Each admitted intake-sets a shared absolute expiry on both pending keys with `PEXPIREAT`, preserving-any later existing expiry using `PEXPIRETIME` (Redis 7+). With no further intake,-both keys expire after the retention period even if every application stops;-idle time is not added to this deadline.+refresh bounds, revision and readiness:++```text+due = last admitted activity + effective project idle+pending expiry = due + recovery grace+```-This is a whole-key inactivity backstop, not native per-member expiry. During-continuous intake, bounded Lua cleanup removes members whose due time is older-than retention; ingestion and dispatcher both perform cleanup.+Tracking uses the project's configured idle duration when available. The default+is 600 seconds outside DEV and 120 seconds in DEV, with the existing environment+override.++Both pending keys receive a whole-key `PEXPIREAT` covering the required expiry,+preserving any existing later deadline with `PEXPIRETIME` (Redis 7+). During+activity, cleanup removes at most 100 members per call with+`due <= now - recovery grace`, using the existing due index. This supports+different project idle durations without another per-member expiry index.+Cleanup runs during ingestion and dispatch; whole-key expiry also clears abandoned+state if all processes stop. Acknowledgement normally removes pending entries+as soon as enqueueing succeeds, independently of their safety deadline.++### Sharding and atomicity++The shard is `SHA256(JSON.stringify([projectId, traceId]))[first 32 bits] % N`.+It is stable for a fixed shard count and hashes the trace identity rather than+the project alone. Redis Cluster then maps the shared `{trace-batch:N}` tag to+one of its 16,384 slots; the number of logical readiness shards is independent+of the number of Redis primaries. Standalone Redis executes the same layout.+Use a `REDIS_KEY_PREFIX` without its own hash tag: Redis uses the first tag in+the full key, so a tagged prefix would collapse every shard onto one slot.++```mermaid+flowchart LR+ identity[Project + trace identity] --> hash[SHA-256 modulo shard count]+ hash --> slot[One Redis Cluster slot per shard tag]+ slot --> due[Due sorted set]+ slot --> state[Pending state hash]+ slot --> first[First-seen bucket hashes]+```++| Operation | Atomic boundary |+| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |+| Track activity | One Lua call reads retained starts and pending state, merges min/max and counters, replaces the revision, sets readiness, moves first-seen history and sets expiry. All keys share a slot. |+| Snapshot/expiry | One Lua call removes a bounded expired cohort and returns readiness counts and the cutoff. |+| Hydrate | One Lua call rechecks due scores and reads corresponding state/revisions. Freshly reactivated traces are not dispatched from an obsolete snapshot. |+| Acknowledge | One Lua call compares the revision, refreshes first-seen retention from dispatch time and removes due/state only when it still matches. |+| Enqueue | A separate BullMQ operation precedes acknowledgement. Stable job IDs reduce duplicates if acknowledgement fails. Concurrent arrivals survive the guarded acknowledgement. |++No application-side read-then-write decides a trace's bounds, revision or due+score. `TIME` before tracking selects the bounded bucket keys; Lua reads Redis+time again and rejects a stale bucket selection before mutating anything.+Scripts serialize within a Redis primary; different primaries can execute+different shards concurrently. Tracking, hydration and acknowledgement calls+contain at most 50 traces; cleanup handles at most 100. Ingestion sends at most+four tracking scripts per pipeline, all in one shard, using one Redis clock lookup+per ingestion call. A group requiring only one script uses a direct call.+Successful pipeline commands remain committed when a sibling+fails; only commands rejected with `NOSCRIPT` are retried using `EVAL`. A pipeline+is not one atomic operation; each Lua call is atomic independently.+Dispatch still pools+up to 1,000 hydrated candidates for locality selection, so script call size does+not constrain queue batch packing. Scripts use cached `EVALSHA`, falling back+to `EVAL` after a missing script cache on the selected primary. Dispatch holds one renewable lease across+all shards, and checks/extends it while making progress. The lease excludes other+dispatchers; it does not lock ingestion. Only each synchronous Redis command or+Lua call blocks its primary, not the dispatcher's entire read/pack/enqueue run.++Lua atomicity prevents command interleaving, but is not an ACID transaction+across Redis, BullMQ and ClickHouse and does not roll back writes on a runtime+script error. Duplicate runs and the accepted Redis failover gap remain part of+this pipeline's delivery contract. There is no reconciler.## Dispatch and reads-One dispatcher holds the renewable lease. It snapshots ready IDs, sorts them by-project, then hydrates and revalidates state in chunks of 1,000. Jobs contain up+One dispatcher holds the renewable lease and snapshots each shard in turn. It+selects the complete due cohort at each shard's cutoff; there is no per-run trace+limit. Each due-ID range read and the combined in-memory candidate list grow+with the due cohort, even though subsequent Lua calls and selection pools are+bounded. It sorts ready IDs by project and due time across shards, then hydrates+and revalidates state in single-slot calls of up to 50. Locality+selection combines those calls into pools of up to 1,000. Jobs contain upto the configured trace cap and weight budgets; project leftovers can share a job. The dispatcherenqueues before acknowledging matching revisions, so concurrent intake stays-pending and failed enqueueing can be retried. Stable job IDs limit duplicates.--### Pending-window size estimates+pending and failed enqueueing can be retried. Successful enqueues are+acknowledged by shard after each hydration window, including when a later+enqueue fails, so Redis calls stay bounded by 50 traces rather than job size.+Stable job IDs limit duplicates.++Acknowledgement removes due/state only. Late arrivals within first-seen+retention reopen readiness with the merged historical and incoming minimum and+maximum starts; the existing two-minute query padding reads the trace again.+Tracking does not classify events as late by comparing timestamps. Every admitted+event updates readiness; retained history supplies the earlier bound even when+the incoming start equals it. History without pending state can also result from+pending expiry, so it does not prove a previous dispatch. Concurrent arrivals+replace the revision and survive an acknowledgement of an older snapshot.++The consumer reads the supplied trace window and checks existing summaries for+each facet. A summary processed since this job was enqueued is skipped on retry;+a later redispatch generates and appends an updated row. There are no Redis+completion markers or staged results. BullMQ still uses Redis for queue delivery.++`ReplacingMergeTree(processed_at)` retains the greatest processing timestamp+when the Topics sort key matches. That key includes the start minute: an event+revealing a genuinely earlier start in a different minute can leave the previous+summary under its original key. `processed_at` is assigned when summary+processing starts, so it orders processing attempts rather than dispatch windows.+An older, narrower dispatch processed later can replace a summary that includes+newer events. This is an accepted limitation; dispatch-order versioning must be+revisited before requiring the newest dispatch window to win.++Retaining the historical maximum preserves the previous tail even when a late+event's start precedes that tail by more than the query buffer. Once historical+bounds expire or are lost, the prior window cannot be reconstructed from this map.++### Trace-window size estimatesAccepted direct-v4 writes contribute their final serialized UTF-8 event size afterfield overflow handling, reusing the writer's existing `event_bytes` calculation.The accounting field itself is excluded. Acceptance means admission to the writer,not confirmation of a ClickHouse flush. `eventUpdateCount` counts accepted updates,-including retries and repeated observation versions; neither counter represents-unique observations or a trace's lifetime total. Deletion after enqueue, readiness-and two-hour retention are unchanged. Later arrivals start another pending window.-Entries missing either counter remain unknown until dispatch or expiry removes them.+including retries and repeated observation versions, rather than unique observations.++Pending state and history carry cumulative estimates for the retained read window.+Tracking adds newly accepted updates to valid pending totals. When reopening a+trace, it initializes totals from retained history before adding the new updates.+Acknowledgement preserves estimates with the bounds, including for oversized+exclusions. A late window therefore cannot make a known oversized trace appear+small or understate its batch weight by counting only its new events.++Missing estimates remain unknown through updates and redispatch until their+retained window expires. They are not replaced with zero or a count of only the+new events. These are admission estimates, not hard limits on actual ClickHouse+rows or query memory; the two-minute query buffer can include additional rows.+Tracking-volume metrics still count only updates accepted in the current ingestion+batch; dispatcher size distributions use the cumulative estimates.The dispatcher emits `estimated_event_update_count` and`estimated_serialized_event_bytes` distributions under `langfuse.trace_batch`, tagged@@ -104,7 +258,7 @@ traces around it into extra jobs. Unknown estimates weigh nothing.Budget-limited jobs report `fill:partial`; the `scope:batch` estimatedistributions show how close jobs run to the budget.-After hydration, a trace whose pending-window estimates exceed+After hydration, a trace whose retained read-window estimates exceed`LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` or`LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` is never queued or read. Thedispatcher acknowledges it like a dispatched trace, increments@@ -113,7 +267,8 @@ dispatcher acknowledges it like a dispatched trace, increments`Trace batch excluded oversized trace` at warn level with `projectId`, `traceId`,`reason` and both estimates. Group that log by `@projectId` to find repeatoffenders. Activity after the acknowledgement starts a new pending window that-is evaluated again, so a trace that keeps growing may be logged more than once.+is evaluated again using its retained cumulative estimates, so a trace that keeps+growing may be logged more than once.Traces with unknown estimates are never excluded.The worker streams full `events_full` payloads and records batch observation,@@ -377,15 +532,15 @@ The following metrics use the `langfuse.trace_batch` prefix. All additions areinside the existing cloud experiment paths; normal ingestion with trackingdisabled performs no new aggregation or Redis calls.-| Metric | Kind / tags | Meaning |-| ---------------------------------------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |-| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|recorded` | Accepted, valid-start-time updates before sampling, and in Redis-acknowledged chunks. Repeated updates count again. Selected volume is `sampling_decisions`. |-| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |-| `failed_read_observation_count`, `failed_read_input_bytes` | Distributions | Partial logical rows and input bytes consumed before a stream failure. Separate from successful throughput. Output and metadata bytes stay on the completion log. |-| `queue_depth` | Gauge; `type:delayed` | Delayed BullMQ jobs. Waiting, active, and failed depth stay on `langfuse.queue.trace_batch.depth`. Request rate, wait time, and processing time use the generic `langfuse.queue.trace_batch` series, the same as every other queue. Processing time is recorded when the processor returns. Discard reasons stay on `discarded_jobs`. |-| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |-| `redis_key_bytes` | Gauge; `key:due\|state` | Estimated memory of each readiness key, including Redis overhead. |-| `redis_key_entries` | Gauge; `key:due\|state` | `ZCARD` / `HLEN` at the memory snapshot. |+| Metric | Kind / tags | Meaning |+| ---------------------------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|recorded` | Accepted, valid-start-time updates before sampling, and in Redis-acknowledged chunks. Repeated updates count again. Selected volume is `sampling_decisions`. |+| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |+| `failed_read_observation_count`, `failed_read_input_bytes` | Distributions | Partial logical rows and input bytes consumed before a stream failure. Separate from successful throughput. Output and metadata bytes stay on the completion log. |+| `queue_depth` | Gauge; `type:delayed` | Delayed BullMQ jobs. Waiting, active, and failed depth stay on `langfuse.queue.trace_batch.depth`. Request rate, wait time, and processing time use the generic `langfuse.queue.trace_batch` series, the same as every other queue. Processing time is recorded when the processor returns. Discard reasons stay on `discarded_jobs`. |+| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |+| `redis_key_bytes` | Gauge; `key:due\|state\|first`, `shard:N\|total` | Estimated memory of each readiness key, including Redis overhead. |+| `redis_key_entries` | Gauge; `key:due\|state\|first`, `shard:N\|total` | `ZCARD` / `HLEN` at the memory snapshot. |The eligible and recorded counters reuse the already-computed serialized size andper-trace aggregates. Recorded volume is emitted after each acknowledged Lua@@ -404,14 +559,17 @@ One failed collection does not suppress the others; failed samples are notreported as zero. Check `langfuse.periodic_runner.completed` and`last_healthy_timestamp_seconds` with `runner:trace_batch_metrics` for freshness.-Memory collection uses one bounded two-key Lua call with `MEMORY USAGE ...+Memory collection uses bounded per-shard Lua calls with `MEMORY USAGE ...SAMPLES 5`, preserving the shared key prefix and Redis Cluster slot routing.Missing keys report zero; there is no keyspace scan or exact full-hash traversal.Memory sampling estimates storage, not process RSS or event payload size.Queue/map snapshots are global and can be emitted by several workers: usemax/latest across reporters, never sum duplicates. For map totals, first-deduplicate reporters per `key`, then add the due/state estimates. In contrast,+deduplicate reporters per `key,shard`, then add shard estimates, or use the+`shard:total` series directly. Never add total and shard series together. First-seen+counts include physically retained expired entries until their bucket expires.+In contrast,`active_reads` is local to a worker; distinct live worker series may be summed.It is emitted on stream start/finish and on each metrics-runner cycle, includingzero when idle, so unchanged long-running reads remain visible.@@ -494,7 +652,7 @@ querying, including failed attempts. The same ID enters ClickHouseunset to suppress these per-attempt logs. Strategy attribution uses thedispatcher metrics described below. Comparequery memory/latency and worker memory as well as input/output/metadata byte-metrics. Redis storage load tests do not measure selector or query memory.+metrics. Redis storage metrics do not account for selector or query memory.### Opt-in locality grouping@@ -554,6 +712,19 @@ and available instance tags in Datadog before relying on these signals.## Stop and drain+Upgrading from the unsharded layout requires a drain; mixed writers and+dispatchers must not run against different layouts. Disable producers, let the+old dispatcher empty `{trace-batch}:due` and `{trace-batch}:state`, and drain the+queue with the old consumers. Stop old dispatchers before deploying all new+producers/dispatchers. Resume tracking only once all use the new layout. There+is no retained first-seen history for traces dispatched before this upgrade.+The same procedure applies before changing shard count or retention. Do not+delete pending keys to simulate a drain. Old queue payloads remain parseable.+Draining pending work does not migrate first-seen history. A shard-count change+must also wait for the old first-seen buckets to expire (TTL plus up to one+bucket), or explicitly accept losing that retained history. Offline migration+of first-seen entries is not implemented here.+Disable ingestion and dispatch to stop new work. Leave the consumer enabled andset the read flag to `false` to complete and remove jobs without querying orparsing their payloads. Jobs at least two hours old are also discarded on pickup.diff --git a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts b/worker/src/features/traceBatching/TraceBatchMetricsRunner.tsindex afa70fa532..b6a1406524 100644--- a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts+++ b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts@@ -7,18 +7,31 @@ import {import { env } from "../../env";import { recordTraceBatchActiveReads } from "../../queues/traceBatchQueue";import { PeriodicRunner } from "../../utils/PeriodicRunner";+import {+ getTraceBatchFirstKeys,+ getTraceBatchReadinessKeys,+} from "./traceBatchReadiness";const METRIC_PREFIX = "langfuse.trace_batch";const MEMORY_INTERVAL_MS = 60_000;-// EVAL applies the shared client's key prefix and routes both keys to their+// EVAL applies the shared client's key prefix and routes keys to their// cluster slot. Raw MEMORY commands do not identify keys in ioredis metadata.const MEMORY_SCRIPT = `-return {+local result = {redis.call('MEMORY', 'USAGE', KEYS[1], 'SAMPLES', 5) or 0,redis.call('ZCARD', KEYS[1]),redis.call('MEMORY', 'USAGE', KEYS[2], 'SAMPLES', 5) or 0,redis.call('HLEN', KEYS[2])}+local bytes = 0+local entries = 0+for i = 3, #KEYS do+ bytes = bytes + (redis.call('MEMORY', 'USAGE', KEYS[i], 'SAMPLES', 5) or 0)+ entries = entries + redis.call('HLEN', KEYS[i])+end+table.insert(result, bytes)+table.insert(result, entries)+return result`;export class TraceBatchMetricsRunner extends PeriodicRunner {@@ -109,24 +122,49 @@ export class TraceBatchMetricsRunner extends PeriodicRunner {private async collectMemory(): Promise<void> {if (!redis) throw new Error("Trace batch metrics require Redis");- const [dueBytes, dueEntries, stateBytes, stateEntries] = (await redis.eval(- MEMORY_SCRIPT,- 2,- "{trace-batch}:due",- "{trace-batch}:state",- )) as number[];+ const clock = await redis.time();+ const now = Number(clock[0]) * 1_000 + Math.floor(Number(clock[1]) / 1_000);+ const totals = [0, 0, 0, 0, 0, 0];+ for (let shard = 0; shard < env.LANGFUSE_TRACE_BATCH_SHARD_COUNT; shard++) {+ const { due, state } = getTraceBatchReadinessKeys(shard);+ const first = getTraceBatchFirstKeys(shard, now);+ const values = (await redis.eval(+ MEMORY_SCRIPT,+ 2 + first.keys.length,+ due,+ state,+ ...first.keys,+ )) as number[];+ this.recordMapMemory(values, String(shard));+ values.forEach((value, index) => (totals[index] += Number(value)));+ }+ this.recordMapMemory(totals, "total");+ }++ private recordMapMemory(values: number[], shard: string): void {+ const [+ dueBytes,+ dueEntries,+ stateBytes,+ stateEntries,+ firstBytes,+ firstEntries,+ ] = values;// These are global snapshots emitted by each eligible worker. Aggregate// duplicate reporters with max/latest, never sum their identical gauges.for (const { key, bytes, entries } of [{ key: "due", bytes: dueBytes, entries: dueEntries },{ key: "state", bytes: stateBytes, entries: stateEntries },+ { key: "first", bytes: firstBytes, entries: firstEntries },]) {recordGauge(`${METRIC_PREFIX}.redis_key_bytes`, bytes, {key,+ shard,unit: "bytes",});recordGauge(`${METRIC_PREFIX}.redis_key_entries`, entries, {key,+ shard,unit: "records",});}diff --git a/worker/src/features/traceBatching/traceBatchReadiness.ts b/worker/src/features/traceBatching/traceBatchReadiness.tsnew file mode 100644index 0000000000..2ee136c0a2--- /dev/null+++ b/worker/src/features/traceBatching/traceBatchReadiness.ts@@ -0,0 +1,35 @@+import { createHash } from "node:crypto";+import { getShardIndex } from "@langfuse/shared/src/server";+import { env } from "../../env";++export function getTraceBatchReadinessKeys(shard: number) {+ const tag = `{trace-batch:${shard}}`;+ return { due: `${tag}:due`, state: `${tag}:state` };+}++export function getTraceBatchShard(member: string): number {+ return getShardIndex(member, env.LANGFUSE_TRACE_BATCH_SHARD_COUNT);+}++// Redis owns the clock. Include the next bucket so a boundary crossed between+// TIME and EVAL still writes into the correct bucket without a retry.+export function getTraceBatchFirstKeys(shard: number, now: number) {+ const ttl = env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS;+ const width = Math.min(3_600_000, ttl);+ const firstBucket = Math.floor((now - ttl) / width);+ const lastBucket = Math.floor(now / width) + 1;+ return {+ width,+ firstBucket,+ keys: Array.from(+ { length: lastBucket - firstBucket + 1 },+ (_, index) => `{trace-batch:${shard}}:first:${firstBucket + index}`,+ ),+ };+}++// A fixed 128-bit identity keeps arbitrary project/trace identifiers out of the+// retained map. Due/state keep the full identifiers for dispatch.+export function getTraceBatchFirstMember(member: string): Buffer {+ return createHash("sha256").update(member).digest().subarray(0, 16);+}diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex afce38c302..7961916c7d 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -16,16 +16,21 @@ import {import { isTopicsProjectEnabled } from "@langfuse/shared/topics/server";import { env } from "../../env";import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";+import {+ getTraceBatchFirstKeys,+ getTraceBatchFirstMember,+ getTraceBatchReadinessKeys,+ getTraceBatchShard,+} from "./traceBatchReadiness";import {getDeterministicSamplingValue,shouldSampleEvaluation,} from "../evaluation/deterministicSampling";-// The shared client applies REDIS_KEY_PREFIX. Both keys must occupy one-// Redis Cluster slot for the atomic update, snapshot, and acknowledgement.-const DUE_KEY = "{trace-batch}:due";-const STATE_KEY = "{trace-batch}:state";const CHUNK_SIZE = 1_000;+const REDIS_CHUNK_SIZE = 50;+const REDIS_PIPELINE_SIZE = 4;+const CLEANUP_SIZE = 100;const QUERY_BUFFER_MS = 2 * 60_000;const MINUTE_MS = 60_000;const NARROW_BATCH_MAX_ENVELOPE_MS = 60 * MINUTE_MS;@@ -579,18 +584,62 @@ const TRACK_SCRIPT = `${EXPIRE_PENDING_SCRIPT}local clock = redis.call('TIME')local now = clock[1] * 1000 + math.floor(clock[2] / 1000)- for i = 3, #ARGV, 6 do+ local firstTtl = tonumber(ARGV[3])+ local width = tonumber(ARGV[4])+ local bucket = math.floor(now / width)+ local current = bucket - tonumber(ARGV[5]) + 3+ if current < 3 or current > #KEYS then+ return redis.error_reply('Trace readiness bucket clock changed; retry intake')+ end+ for i = 6, #ARGV, 7 dolocal member = ARGV[i]local minStart = tonumber(ARGV[i + 1])local maxStart = tonumber(ARGV[i + 2])local eventUpdateCount = tonumber(ARGV[i + 4])local serializedEventBytes = tonumber(ARGV[i + 5])+ local identity = ARGV[i + 6]local previous = redis.call('HGET', KEYS[2], member)- if previous then- local state = cjson.decode(previous)+ local state = previous and cjson.decode(previous)+ local priorDue = tonumber(redis.call('ZSCORE', KEYS[1], member))+ if not priorDue or priorDue <= now - tonumber(ARGV[2]) then state = nil end+ if state and state.historyBucket then+ local old = state.historyBucket - tonumber(ARGV[5]) + 3+ if old >= 3 and old <= #KEYS and old ~= current then+ redis.call('HDEL', KEYS[old], identity)+ end+ else+ local retainedUpdates = 0+ local retainedBytes = 0+ local estimatesKnown = true+ for key = 3, #KEYS do+ local retained = redis.call('HGET', KEYS[key], identity)+ if retained then+ local bounds = cmsgpack.unpack(retained)+ if bounds[3] > now then+ minStart = math.min(minStart, bounds[1])+ maxStart = math.max(maxStart, bounds[2])+ if bounds[4] and bounds[5] then+ retainedUpdates = math.max(retainedUpdates, bounds[4])+ retainedBytes = math.max(retainedBytes, bounds[5])+ else+ estimatesKnown = false+ end+ end+ if key ~= current then redis.call('HDEL', KEYS[key], identity) end+ end+ end+ if not estimatesKnown then+ eventUpdateCount = nil+ serializedEventBytes = nil+ elseif not state then+ eventUpdateCount = eventUpdateCount + retainedUpdates+ serializedEventBytes = serializedEventBytes + retainedBytes+ end+ end+ if state thenminStart = math.min(minStart, state.minStart)maxStart = math.max(maxStart, state.maxStart)- if state.eventUpdateCount and state.serializedEventBytes then+ if eventUpdateCount and serializedEventBytes and state.eventUpdateCount and state.serializedEventBytes theneventUpdateCount = eventUpdateCount + state.eventUpdateCountserializedEventBytes = serializedEventBytes + state.serializedEventByteselse@@ -600,19 +649,24 @@ const TRACK_SCRIPT = `endendredis.call('HSET', KEYS[2], member, cjson.encode({- minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3],+ minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3], historyBucket = bucket,eventUpdateCount = eventUpdateCount, serializedEventBytes = serializedEventBytes}))redis.call('ZADD', KEYS[1], now + tonumber(ARGV[1]), member)+ redis.call('HSET', KEYS[current], identity,+ cmsgpack.pack({minStart, maxStart, now + firstTtl, eventUpdateCount, serializedEventBytes}))end+ local firstExpiry = math.max((bucket + 1) * width + firstTtl,+ redis.call('PEXPIRETIME', KEYS[current]))+ -- Integer command arguments need decimal strings, including round timestamps.+ redis.call('PEXPIREAT', KEYS[current], string.format('%.0f', firstExpiry))local retention = tonumber(ARGV[2])- local expired = expirePending(now, retention, ${CHUNK_SIZE})- -- Bound abandoned state from the last admitted activity, independent of idle.- -- A shorter policy on another writer must not shorten an existing expiry.- local expiresAt = math.max(now + retention,+ local expired = expirePending(now, retention, ${CLEANUP_SIZE})+ -- The shared inactivity backstop covers the latest required member deadline.+ local expiresAt = math.max(now + tonumber(ARGV[1]) + retention,redis.call('PEXPIRETIME', KEYS[1]), redis.call('PEXPIRETIME', KEYS[2]))- redis.call('PEXPIREAT', KEYS[1], expiresAt)- redis.call('PEXPIREAT', KEYS[2], expiresAt)+ redis.call('PEXPIREAT', KEYS[1], string.format('%.0f', expiresAt))+ redis.call('PEXPIREAT', KEYS[2], string.format('%.0f', expiresAt))return expired`;@@ -624,7 +678,7 @@ const SNAPSHOT_SCRIPT = `local now = clock[1] * 1000 + math.floor(clock[2] / 1000)local retention = tonumber(ARGV[1])local cutoff = tonumber(ARGV[2]) or now- local expired = expirePending(now, retention, ${CHUNK_SIZE})+ local expired = expirePending(now, retention, ${CLEANUP_SIZE})local pending = redis.call('ZCARD', KEYS[1])local ready = math.max(0, redis.call('ZCOUNT', KEYS[1], '-inf', cutoff)- redis.call('ZCOUNT', KEYS[1], '-inf', now - retention))@@ -658,10 +712,32 @@ const HYDRATE_SCRIPT = `// Returns one flag per member, in argument order: 1 when removed, 0 when a// newer revision keeps the trace pending.const ACKNOWLEDGE_SCRIPT = `+ local clock = redis.call('TIME')+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)+ local width = tonumber(ARGV[2])+ local bucket = math.floor(now / width)+ local current = bucket - tonumber(ARGV[3]) + 3+ if current < 3 or current > #KEYS then+ return redis.error_reply('Trace readiness bucket clock changed; retry acknowledgement')+ endlocal removed = {}- for i = 1, #ARGV, 2 do+ local retained = false+ for i = 4, #ARGV, 3 dolocal state = redis.call('HGET', KEYS[2], ARGV[i])- if state and cjson.decode(state).revision == ARGV[i + 1] then+ local decoded = state and cjson.decode(state)+ if decoded and decoded.revision == ARGV[i + 1] then+ local identity = ARGV[i + 2]+ if decoded.historyBucket then+ local old = decoded.historyBucket - tonumber(ARGV[3]) + 3+ if old >= 3 and old <= #KEYS and old ~= current then redis.call('HDEL', KEYS[old], identity) end+ else+ for key = 3, #KEYS do+ if key ~= current then redis.call('HDEL', KEYS[key], identity) end+ end+ end+ redis.call('HSET', KEYS[current], identity,+ cmsgpack.pack({decoded.minStart, decoded.maxStart, now + tonumber(ARGV[1]), decoded.eventUpdateCount, decoded.serializedEventBytes}))+ retained = trueredis.call('ZREM', KEYS[1], ARGV[i])redis.call('HDEL', KEYS[2], ARGV[i])table.insert(removed, 1)@@ -669,9 +745,42 @@ const ACKNOWLEDGE_SCRIPT = `table.insert(removed, 0)endend+ if retained then+ local expiresAt = math.max((bucket + 1) * width + tonumber(ARGV[1]),+ redis.call('PEXPIRETIME', KEYS[current]))+ redis.call('PEXPIREAT', KEYS[current], string.format('%.0f', expiresAt))+ endreturn removed`;+const readinessScriptHashes = new Map(+ [TRACK_SCRIPT, SNAPSHOT_SCRIPT, HYDRATE_SCRIPT, ACKNOWLEDGE_SCRIPT].map(+ (script) =>+ [script, createHash("sha1").update(script).digest("hex")] as const,+ ),+);++async function evalReadiness(+ script: string,+ keys: string[],+ args: (string | number | Buffer)[],+): Promise<unknown> {+ if (!redis) throw new Error("Trace batching requires Redis");+ try {+ return await redis.evalsha(+ readinessScriptHashes.get(script)!,+ keys.length,+ ...keys,+ ...args,+ );+ } catch (error) {+ // SCRIPT FLUSH or a promoted primary can have an empty script cache.+ if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT"))+ throw error;+ return redis.eval(script, keys.length, ...keys, ...args);+ }+}+function recordTrackingVolume(stage: "eligible" | "recorded",entries: Iterable<@@ -700,6 +809,7 @@ export async function trackTraceBatchActivity(startTimeISO: string;serializedEventBytes: number;}[],+ idleMs?: number,): Promise<void> {if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||@@ -758,36 +868,106 @@ export async function trackTraceBatchActivity({ decision: "excluded" },);if (entries.length === 0) return;+ const effectiveIdleMs = idleMs ?? (await resolveTraceIdleMs(projectId));+ if (!Number.isSafeInteger(effectiveIdleMs) || effectiveIdleMs <= 0)+ throw new Error("Trace batch idle time must be a positive integer");if (!redis) throw new Error("Trace batching requires Redis");- const idleMs = await resolveTraceIdleMs(projectId);- // Bounded scripts keep ingestion from monopolizing the global Redis slot.- for (let offset = 0; offset < entries.length; offset += CHUNK_SIZE) {- const chunk = entries.slice(offset, offset + CHUNK_SIZE);- const expired = Number(- await redis.eval(- TRACK_SCRIPT,- 2,- DUE_KEY,- STATE_KEY,- idleMs,- env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,- ...chunk.flatMap(([traceId, state]) => [- JSON.stringify([projectId, traceId]),- state.minStart,- state.maxStart,- // Unique even after dispatch deletes and a later arrival recreates state.- randomUUID(),- state.eventUpdateCount,- state.serializedEventBytes,- ]),- ),- );- // Count only acknowledged chunks; a timeout can leave Redis's outcome unknown.- recordTrackingVolume("recorded", chunk);- recordIncrement("langfuse.trace_batch.expired_traces", expired, {- source: "ingestion",- });- recordIncrement("langfuse.trace_batch.tracked_traces", chunk.length);+ const shards = new Map<number, typeof entries>();+ for (const entry of entries) {+ const shard = getTraceBatchShard(JSON.stringify([projectId, entry[0]]));+ const group = shards.get(shard) ?? [];+ group.push(entry);+ shards.set(shard, group);+ }+ const clock = await redis.time();+ const now = Number(clock[0]) * 1_000 + Math.floor(Number(clock[1]) / 1_000);+ // Each pipeline contains bounded scripts from one cluster slot.+ for (const [shard, group] of shards) {+ const { due, state: stateKey } = getTraceBatchReadinessKeys(shard);+ const first = getTraceBatchFirstKeys(shard, now);+ const keys = [due, stateKey, ...first.keys];+ for (+ let offset = 0;+ offset < group.length;+ offset += REDIS_CHUNK_SIZE * REDIS_PIPELINE_SIZE+ ) {+ const pipeline =+ group.length - offset > REDIS_CHUNK_SIZE+ ? redis.pipeline()+ : undefined;+ const commands: {+ chunk: typeof entries;+ args: (string | number | Buffer)[];+ }[] = [];+ for (+ let start = offset;+ start <+ Math.min(+ group.length,+ offset + REDIS_CHUNK_SIZE * REDIS_PIPELINE_SIZE,+ );+ start += REDIS_CHUNK_SIZE+ ) {+ const chunk = group.slice(start, start + REDIS_CHUNK_SIZE);+ const args = [+ effectiveIdleMs,+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS,+ first.width,+ first.firstBucket,+ ...chunk.flatMap(([traceId, state]) => [+ JSON.stringify([projectId, traceId]),+ state.minStart,+ state.maxStart,+ // Unique even after dispatch deletes and a later arrival recreates state.+ randomUUID(),+ state.eventUpdateCount,+ state.serializedEventBytes,+ getTraceBatchFirstMember(JSON.stringify([projectId, traceId])),+ ]),+ ];+ pipeline?.evalsha(+ readinessScriptHashes.get(TRACK_SCRIPT)!,+ keys.length,+ ...keys,+ ...args,+ );+ commands.push({ chunk, args });+ }+ const results: [Error | null, unknown][] | null = pipeline+ ? await pipeline.exec()+ : [[null, await evalReadiness(TRACK_SCRIPT, keys, commands[0].args)]];+ if (!results)+ throw new Error("Trace batch tracking pipeline returned no results");+ let failure: unknown;+ // A command error does not undo successful siblings. Record every known result.+ for (const [index, { chunk, args }] of commands.entries()) {+ try {+ const [error, result] = results[index];+ let expired: number;+ if (error) {+ if (!error.message.startsWith("NOSCRIPT")) throw error;+ // Only this cache miss is safe to retry; successful siblings are not replayed.+ expired = Number(+ await redis.eval(TRACK_SCRIPT, keys.length, ...keys, ...args),+ );+ } else {+ expired = Number(result);+ }+ recordTrackingVolume("recorded", chunk);+ recordIncrement("langfuse.trace_batch.expired_traces", expired, {+ source: "ingestion",+ });+ recordIncrement(+ "langfuse.trace_batch.tracked_traces",+ chunk.length,+ );+ } catch (error) {+ if (failure === undefined) failure = error;+ }+ }+ if (failure !== undefined) throw failure;+ }}} catch (error) {// This optional load experiment must not fail normal event ingestion.@@ -884,59 +1064,91 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {if (!redis) throw new Error("Trace batching requires Redis");const queue = TraceBatchQueue.getInstance();if (!queue) throw new Error("Trace batch queue is unavailable");- let cutoff = "";- let now = 0;- while (!this.stopping) {- await this.extendLockOnProgress();- const snapshot = (await redis.eval(- SNAPSHOT_SCRIPT,- 2,- DUE_KEY,- STATE_KEY,- env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,+ const totals = { pending: 0, ready: 0, oldestAge: 0 };+ const members: {+ member: string;+ projectId: string;+ shard: number;+ cutoff: string;+ due: number;+ }[] = [];+ for (let shard = 0; shard < env.LANGFUSE_TRACE_BATCH_SHARD_COUNT; shard++) {+ if (this.stopping) return;+ const { due, state } = getTraceBatchReadinessKeys(shard);+ let cutoff = "";+ let now = 0;+ while (!this.stopping) {+ await this.extendLockOnProgress();+ const snapshot = (await evalReadiness(+ SNAPSHOT_SCRIPT,+ [due, state],+ [env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS, cutoff],+ )) as (string | number)[];+ const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);+ now = clock;+ cutoff = String(snapshot[1]);+ recordIncrement("langfuse.trace_batch.expired_traces", expired, {+ source: "dispatcher",+ });+ recordGauge("langfuse.trace_batch.pending_traces", pending, {+ shard: String(shard),+ });+ recordGauge("langfuse.trace_batch.ready_traces", ready, {+ shard: String(shard),+ });+ recordGauge(+ "langfuse.trace_batch.oldest_due_age_ms",+ Math.max(0, now - oldest),+ { shard: String(shard) },+ );+ if (expired < CLEANUP_SIZE) {+ totals.pending += pending;+ totals.ready += ready;+ totals.oldestAge = Math.max(+ totals.oldestAge,+ Math.max(0, now - oldest),+ );+ break;+ }+ }+ if (this.stopping) return;+ const dueMembers = await redis.zrange(+ due,+ `(${now - env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS}`,cutoff,- )) as (string | number)[];- const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);- now = clock;- cutoff = String(snapshot[1]);- recordIncrement("langfuse.trace_batch.expired_traces", expired, {- source: "dispatcher",- });- recordGauge("langfuse.trace_batch.pending_traces", pending);- recordGauge("langfuse.trace_batch.ready_traces", ready);- recordGauge(- "langfuse.trace_batch.oldest_due_age_ms",- Math.max(0, now - oldest),+ "BYSCORE",+ "WITHSCORES",);- if (expired < CHUNK_SIZE) break;+ recordDistribution(+ "langfuse.trace_batch.snapshot_size",+ dueMembers.length / 2,+ );+ for (let index = 0; index < dueMembers.length; index += 2) {+ const member = dueMembers[index];+ members.push({+ member,+ projectId: (JSON.parse(member) as [string, string])[0],+ shard,+ cutoff,+ due: Number(dueMembers[index + 1]),+ });+ }}+ recordGauge("langfuse.trace_batch.pending_traces", totals.pending);+ recordGauge("langfuse.trace_batch.ready_traces", totals.ready);+ recordGauge("langfuse.trace_batch.oldest_due_age_ms", totals.oldestAge);if (this.stopping) return;- // One range read captures all due IDs without cursor races or loading every- // trace's state. Memory and the Redis response size grow with the due cohort.+ // Each range read captures one shard's due IDs. Pack the combined cohort so+ // Redis placement does not determine the ClickHouse query boundaries.const strategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;- const members = (- await redis.zrange(- DUE_KEY,- `(${now - env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS}`,- cutoff,- "BYSCORE",- )- ).map((member) => ({- member,- projectId: (JSON.parse(member) as [string, string])[0],- }));- // Stable sorting keeps Redis readiness order within each project.- members.sort((a, b) => {- if (a.projectId < b.projectId) {- return -1;- }- if (a.projectId > b.projectId) {- return 1;- }- return 0;- });- recordDistribution("langfuse.trace_batch.snapshot_size", members.length);+ // Project packing needs due order within each project, including across shards.+ members.sort(+ (a, b) =>+ compareStrings(a.projectId, b.projectId) ||+ compareNumbers(a.due, b.due) ||+ compareStrings(a.member, b.member),+ );const batchWeightLimits: TraceBatchWeightLimits = {maxEventUpdates: toLimit(env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES,@@ -956,16 +1168,62 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {const acknowledge = async (batch: readonly PendingTrace[],- ): Promise<boolean[]> =>- (- (await redis!.eval(- ACKNOWLEDGE_SCRIPT,- 2,- DUE_KEY,- STATE_KEY,- ...batch.flatMap(({ member, trace }) => [member, trace.revision]),- )) as number[]- ).map((flag) => Number(flag) === 1);+ ): Promise<boolean[]> => {+ const removed = Array<boolean>(batch.length).fill(false);+ const groups = new Map<+ number,+ { entry: PendingTrace; index: number }[]+ >();+ batch.forEach((entry, index) => {+ const shard = getTraceBatchShard(entry.member);+ const group = groups.get(shard) ?? [];+ group.push({ entry, index });+ groups.set(shard, group);+ });+ for (const [shard, group] of groups) {+ const { due, state } = getTraceBatchReadinessKeys(shard);+ for (+ let offset = 0;+ offset < group.length;+ offset += REDIS_CHUNK_SIZE+ ) {+ const chunk = group.slice(offset, offset + REDIS_CHUNK_SIZE);+ const clock = await redis!.time();+ const first = getTraceBatchFirstKeys(+ shard,+ Number(clock[0]) * 1_000 + Math.floor(Number(clock[1]) / 1_000),+ );+ const flags = (await evalReadiness(+ ACKNOWLEDGE_SCRIPT,+ [due, state, ...first.keys],+ [+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS,+ first.width,+ first.firstBucket,+ ...chunk.flatMap(({ entry: { member, trace } }) => [+ member,+ trace.revision,+ getTraceBatchFirstMember(member),+ ]),+ ],+ )) as number[];+ chunk.forEach(({ index }, position) => {+ removed[index] = Number(flags[position]) === 1;+ });+ }+ }+ return removed;+ };++ const enqueued: PendingTrace[] = [];+ const flushEnqueued = async () => {+ if (enqueued.length === 0) return;+ const removed = await acknowledge(enqueued.splice(0));+ recordIncrement(+ "langfuse.trace_batch.reactivated_traces",+ removed.filter((wasRemoved) => !wasRemoved).length,+ );+ };// Excluded traces leave Redis like dispatched ones; activity after this// run's snapshot starts a fresh pending window that is evaluated again.@@ -1017,6 +1275,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {},{ jobId: id },);+ enqueued.push(...batch);recordDistribution("langfuse.trace_batch.size", batch.length);recordDistribution("langfuse.trace_batch.project_count",@@ -1088,11 +1347,6 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {strategy,});}- const removed = await acknowledge(batch);- recordIncrement(- "langfuse.trace_batch.reactivated_traces",- removed.filter((wasRemoved) => !wasRemoved).length,- );};let projectBatchTail: PendingTrace[] = [];@@ -1104,32 +1358,54 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {) {await this.extendLockOnProgress();const candidates = members.slice(offset, offset + CHUNK_SIZE);- const hydrated = (await redis.eval(- HYDRATE_SCRIPT,- 2,- DUE_KEY,- STATE_KEY,- cutoff,- env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,- ...candidates.map(({ member }) => member),- )) as string[];+ const byShard = new Map<number, typeof candidates>();+ for (const candidate of candidates) {+ const group = byShard.get(candidate.shard) ?? [];+ group.push(candidate);+ byShard.set(candidate.shard, group);+ }+ const hydratedByMember = new Map<string, [string, string]>();+ for (const [shard, group] of byShard) {+ const { due, state } = getTraceBatchReadinessKeys(shard);+ for (let start = 0; start < group.length; start += REDIS_CHUNK_SIZE) {+ await this.extendLockOnProgress();+ const part = (await evalReadiness(+ HYDRATE_SCRIPT,+ [due, state],+ [+ group[0].cutoff,+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,+ ...group+ .slice(start, start + REDIS_CHUNK_SIZE)+ .map(({ member }) => member),+ ],+ )) as string[];+ for (let index = 0; index < part.length; index += 3) {+ hydratedByMember.set(part[index], [+ part[index + 1],+ part[index + 2],+ ]);+ }+ }+ }recordIncrement("langfuse.trace_batch.skipped_traces",- candidates.length - hydrated.length / 3,+ candidates.length - hydratedByMember.size,);const hydratedCandidates: PendingTrace[] = [];const excluded: {entry: PendingTrace;reason: TraceExclusionReason;}[] = [];- for (let i = 0; i < hydrated.length; i += 3) {+ for (const { member } of candidates) {if (this.stopping) return;- const member = hydrated[i];+ const hydrated = hydratedByMember.get(member);+ if (!hydrated) continue;const [projectId, traceId] = JSON.parse(member) as [string, string];- const state = JSON.parse(hydrated[i + 1]);+ const state = JSON.parse(hydrated[0]);const entry: PendingTrace = {member,- due: Number(hydrated[i + 2]),+ due: Number(hydrated[1]),estimates:Number.isSafeInteger(state.eventUpdateCount) &&state.eventUpdateCount >= 0 &&@@ -1179,49 +1455,59 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {{ strategy },);- if (strategy === "project") {- projectBatchTail = [];- if (selected.at(-1)?.length !== env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {- projectBatchTail = selected.pop() ?? [];+ try {+ if (strategy === "project") {+ projectBatchTail = [];+ if (selected.at(-1)?.length !== env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {+ projectBatchTail = selected.pop() ?? [];+ }+ for (const batch of selected) {+ if (this.stopping) return;+ await enqueue(batch);+ }+ continue;+ }++ const fullBatches = selected.filter(+ (batch) => batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ );+ const partialBatches = selected.filter(+ (batch) => batch.length < env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ );+ for (const batch of fullBatches) {+ if (this.stopping) return;+ await enqueue(batch);}- for (const batch of selected) {+ await this.extendLockOnProgress();+ if (this.stopping) return;+ const prepared = prepareLocalityPartials(+ partialBatches,+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ batchWeightLimits,+ );+ localityPartials = prepared.carry;+ for (const batch of prepared.dispatch) {if (this.stopping) return;await enqueue(batch);}- continue;+ } finally {+ await flushEnqueued();}-- const fullBatches = selected.filter(- (batch) => batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE,- );- const partialBatches = selected.filter(- (batch) => batch.length < env.LANGFUSE_TRACE_BATCH_MAX_SIZE,- );- for (const batch of fullBatches) {- if (this.stopping) return;- await enqueue(batch);+ }+ if (this.stopping) return;+ try {+ if (strategy === "project") {+ if (projectBatchTail.length > 0) await enqueue(projectBatchTail);+ return;}- await this.extendLockOnProgress();- if (this.stopping) return;- const prepared = prepareLocalityPartials(- partialBatches,- env.LANGFUSE_TRACE_BATCH_MAX_SIZE,- batchWeightLimits,- );- localityPartials = prepared.carry;- for (const batch of prepared.dispatch) {+ for (const batch of localityPartials.toSorted(+ compareBatchesByOldestDue,+ )) {if (this.stopping) return;await enqueue(batch);}- }- if (this.stopping) return;- if (strategy === "project") {- if (projectBatchTail.length > 0) await enqueue(projectBatchTail);- return;- }- for (const batch of localityPartials.toSorted(compareBatchesByOldestDue)) {- if (this.stopping) return;- await enqueue(batch);+ } finally {+ await flushEnqueued();}}}diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex 6c2423ad95..8adea51fdd 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -97,6 +97,7 @@ async function loadProjectRule(projectId: string) {async function summarizeTraceBatch(observations: Observation[],topics: TopicsBatchProjects,+ jobEnqueuedAt: Date,): Promise<TraceOutcome> {const first = observations[0];const traceId = first?.traceId;@@ -130,6 +131,7 @@ async function summarizeTraceBatch(projectId: first.projectId,traceId,traceTimestamp,+ jobEnqueuedAt,environment: first.environment,traceName: first.name ?? "",transcript,@@ -316,7 +318,7 @@ export const traceBatchQueueProcessor: Processor<);if (!summaryFailed)topicsOutcomes.push(- await summarizeTraceBatch(observations, topics),+ await summarizeTraceBatch(observations, topics, event.timestamp),);}).catch((error: unknown) => {