Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(trace-batching): reprocess late arrivals with sharded readiness (#18602)

Lysander Kiesel6069e090ee9d
diff --git a/.env.dev.example b/.env.dev.example
index 72ab869c2d..1346352018 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -93,6 +93,10 @@ NEXTAUTH_SECRET="secret"
# LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES=104857600
# Unset idle is 2 minutes when NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=DEV, else 10 minutes.
LANGFUSE_TRACE_BATCH_IDLE_MS=120000
+# Internal cloud readiness retention (at most 48h) and sharding; drain before changing.
+# LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS=86400000
+# Optional override; unset uses 16 on Redis Cluster and 1 on standalone/Sentinel.
+# LANGFUSE_TRACE_BATCH_SHARD_COUNT=16
# Summaries and embeddings use Bedrock in LANGFUSE_AI_AWS_BEDROCK_REGION.
# Optional local profile: LANGFUSE_AI_FEATURES_AWS_PROFILE (AWS_PROFILE wins).
# Postgres, ClickHouse, and Redis are the same local services as the rest of the stack.
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index 9cfdd34018..0042823fef 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -30,6 +30,8 @@
- Repository layer: `src/server/repositories/*`
- Queue payload schemas: `src/server/queues.ts`
- Queue helpers: `src/server/redis/*`
+ `getShardIndex` from `src/server/redis/sharding.ts` is exported by the server
+ barrel; hash the unit identity when a project can dominate traffic.
- Internal trace-batch queue: `src/server/redis/traceBatch.ts` (cloud-gated);
payloads in `src/server/queues.ts` accept persisted single-project jobs.
Full-event streaming reads live in `src/server/repositories/trace-batch.ts`;
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index d7a3bffdf4..b17fad518c 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -110,6 +110,7 @@ export * from "./redis/monitorQueue";
export * from "./redis/inAppAgentRunQueue";
export * from "./redis/traceDelete";
export * from "./redis/traceBatch";
+export * from "./redis/sharding";
export * from "./redis/projectDelete";
export * from "./redis/scoreDelete";
export * from "./redis/datasetDelete";
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index fa762dc2d6..b78cd131eb 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -50,6 +50,26 @@
`src/features/traceBatching/TraceBatchMetricsRunner.ts` collects bounded queue
and Redis snapshots independently of dispatch/consumption when either role is
enabled. Global snapshot gauges must not be summed across worker reporters.
+ Readiness uses project/trace-hashed shards with one dispatcher lease; all Lua
+ keys share a shard hash tag. Defaults are 16 shards on Redis Cluster and 1 on
+ standalone/Sentinel, with an explicit count override. Redis resizing does not
+ change the count. Hourly first-seen hashes retain packed minimum and
+ maximum event starts and cumulative size estimates after acknowledgement for late arrivals;
+ history retention defaults to 24 hours and is capped at 48 hours. Pending state
+ carries the bucket pointer. Pending expiry is due plus recovery grace, including
+ the whole-key backstop. Tracking uses the project's configured idle duration,
+ falling back to the environment default when unavailable.
+ Cached Lua calls cap tracking/hydration/acknowledgement at 50 and cleanup at 100;
+ ingestion pipelines at most four scripts within a shard, with one clock lookup
+ per ingestion call; single-script groups use a direct call. Record all
+ acknowledged siblings after a pipeline error; retry
+ only `NOSCRIPT` commands. Retained estimates guard the whole known read window,
+ including late arrivals; unknown estimates remain unknown. Locality selection
+ still pools 1,000 candidates. Drain the old layout before upgrade
+ or shard-count/retention changes. The trace-batch consumer checks existing
+ summaries against the job enqueue time so retries skip completed facets while
+ later redispatches append updated summaries. Map gauges include shard and
+ total series; never sum duplicate reporters or add totals to shard series.
`src/features/traceBatching/traceBatchTranscript.ts` measures per-trace assembly
phases, thread count, current-turn/history token estimates and their sum. Each
admitted trace records full assembled-transcript JSON lengths, renders the
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
index 092a962689..b3cf129b78 100644
--- a/worker/src/__tests__/traceBatching.test.ts
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -29,6 +29,11 @@ import * as shared from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { env } from "../env";
import { TraceBatchMetricsRunner } from "../features/traceBatching/TraceBatchMetricsRunner";
+import {
+ getTraceBatchFirstKeys,
+ getTraceBatchReadinessKeys,
+ getTraceBatchShard,
+} from "../features/traceBatching/traceBatchReadiness";
import {
prepareLocalityPartials,
selectTraceBatches,
@@ -39,6 +44,15 @@ import {
import { otelIngestionQueueProcessorBuilder } from "../queues/otelIngestionQueue";
import { traceBatchQueueProcessor } from "../queues/traceBatchQueue";
import { ClickhouseWriter } from "../services/ClickhouseWriter";
+import {
+ ensureDefaultTopicFacets,
+ listTopicSummaries,
+ type TopicsModels,
+} from "@langfuse/shared/topics/server";
+import {
+ summarizeTopicTraceFacets,
+ embedTopicSummary,
+} from "../features/topics/models";
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const original =
@@ -60,6 +74,41 @@ const topicsProjects = vi.hoisted(() => new Set<string>());
vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({
...(await importOriginal<typeof import("@langfuse/shared/topics/server")>()),
isTopicsProjectEnabled: (projectId: string) => topicsProjects.has(projectId),
+ getEnabledTopicsModels: async (projectIds: string[]) =>
+ new Map(
+ projectIds
+ .filter((id) => topicsProjects.has(id))
+ .map((projectId) => {
+ const slot = (
+ name: "summary" | "embedding" | "clustering",
+ model: string,
+ ) => ({
+ slot: name,
+ provider: "bedrock",
+ adapter: "bedrock",
+ model,
+ connection: { secretKey: "encrypted" },
+ });
+ return [
+ projectId,
+ {
+ projectId,
+ enabled: true,
+ summary: slot("summary", "us.openai.gpt-5.6-luna"),
+ embedding: {
+ ...slot("embedding", "eu.cohere.embed-v4:0"),
+ dimensions: 1024,
+ },
+ clustering: slot("clustering", "us.openai.gpt-5.6-terra"),
+ } as TopicsModels,
+ ] as const;
+ }),
+ ),
+}));
+
+vi.mock("../features/topics/models", () => ({
+ summarizeTopicTraceFacets: vi.fn(),
+ embedTopicSummary: vi.fn(),
}));
vi.mock("../features/evaluation/observationEval", async (importOriginal) => ({
@@ -526,8 +575,7 @@ describe("trace batch selection", () => {
});
describe("trace micro-batch scheduling with Redis", () => {
- const dueKey = "{trace-batch}:due";
- const stateKey = "{trace-batch}:state";
+ const { due: dueKey, state: stateKey } = getTraceBatchReadinessKeys(0);
const originalEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
@@ -539,6 +587,8 @@ describe("trace micro-batch scheduling with Redis", () => {
const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;
const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;
const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;
+ const originalFirstTtl = env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS;
+ const originalShards = env.LANGFUSE_TRACE_BATCH_SHARD_COUNT;
const originalMaxTraceEventUpdates =
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES;
const originalMaxTraceBytes =
@@ -575,6 +625,18 @@ describe("trace micro-batch scheduling with Redis", () => {
);
}
+ async function clearReadiness() {
+ for (let shard = 0; shard < env.LANGFUSE_TRACE_BATCH_SHARD_COUNT; shard++) {
+ const keys = getTraceBatchReadinessKeys(shard);
+ await client().del(
+ keys.due,
+ keys.state,
+ ...getTraceBatchFirstKeys(shard, Date.now()).keys,
+ );
+ }
+ await client().del("{trace-batch}:dispatcher");
+ }
+
beforeEach(async () => {
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "true";
@@ -586,9 +648,11 @@ describe("trace micro-batch scheduling with Redis", () => {
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 60;
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 600_000;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 7_200_000;
+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 86_400_000;
+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 1;
// The unset default is 2 minutes on DEV; pin the production idle time.
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 600_000;
- await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");
+ await clearReadiness();
const redisConnection = createNewRedisInstance();
if (!redisConnection) throw new Error("Redis is required for this test");
connection = redisConnection;
@@ -603,6 +667,7 @@ describe("trace micro-batch scheduling with Redis", () => {
runners.length = 0;
vi.restoreAllMocks();
vi.clearAllMocks();
+ await clearReadiness();
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = originalEnabled;
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;
@@ -613,16 +678,240 @@ describe("trace micro-batch scheduling with Redis", () => {
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;
env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;
+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = originalFirstTtl;
+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = originalShards;
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES =
originalMaxTraceEventUpdates;
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES = originalMaxTraceBytes;
await queue.obliterate({ force: true });
await queue.close();
connection.disconnect();
- await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");
});
afterAll(() => client().disconnect());
+ it("re-dispatches late arrivals with the earliest start of the whole trace", async () => {
+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 600;
+ const start = Date.now() - 20 * 60_000;
+ await trackTraceBatchActivity("project", [event("late-trace", start)]);
+ const trackedAt = Date.now();
+ await vi.waitFor(() =>
+ expect(Date.now() - trackedAt).toBeGreaterThanOrEqual(400),
+ );
+ await makeDue("project", "late-trace");
+ await runner().processBatch();
+ const [first] = await queue.getJobs(["wait"]);
+ expect(first.data.payload.traces[0].minStart).toBe(start);
+ expect(await client().zcard(dueKey)).toBe(0);
+ const dispatchedAt = Date.now();
+ await vi.waitFor(() =>
+ expect(Date.now() - dispatchedAt).toBeGreaterThanOrEqual(300),
+ );
+
+ const lateStart = start + 10 * 60_000;
+ await trackTraceBatchActivity("project", [event("late-trace", lateStart)]);
+ await makeDue("project", "late-trace");
+ await runner().processBatch();
+ const jobs = await queue.getJobs(["wait"]);
+ expect(jobs).toHaveLength(2);
+ const later = jobs.find((job) => job.id !== first.id)!;
+ expect(later.data.payload.traces[0]).toMatchObject({
+ minStart: start,
+ maxStart: lateStart,
+ });
+ expect(later.data.payload.traces[0].revision).not.toBe(
+ first.data.payload.traces[0].revision,
+ );
+ const backdatedStart = start + 3 * 60_000;
+ await trackTraceBatchActivity("project", [
+ event("late-trace", backdatedStart),
+ ]);
+ await makeDue("project", "late-trace");
+ await runner().processBatch();
+ const backdated = (await queue.getJobs(["wait"])).find(
+ (job) => job.id !== first.id && job.id !== later.id,
+ )!;
+ expect(backdated.data.payload.traces[0]).toMatchObject({
+ minStart: start,
+ maxStart: lateStart,
+ });
+ });
+
+ it("keeps abandoned pending keys through idle time and the recovery grace", async () => {
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 200;
+ await trackTraceBatchActivity("project", [event("long-idle")], 500);
+ const due = Number(
+ await client().zscore(dueKey, member("project", "long-idle")),
+ );
+ for (const key of [dueKey, stateKey])
+ expect(Number(await client().pexpiretime(key))).toBeGreaterThanOrEqual(
+ due + 200,
+ );
+ await vi.waitFor(() =>
+ expect(Date.now()).toBeGreaterThanOrEqual(due + 100),
+ );
+ expect(await client().exists(dueKey, stateKey)).toBe(2);
+ await vi.waitFor(async () =>
+ expect(await client().exists(dueKey, stateKey)).toBe(0),
+ );
+ });
+
+ it("expires first-seen history natively while keeping pending state independently", async () => {
+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 150;
+ const first = Date.now() - 600_000;
+ await trackTraceBatchActivity("project", [event("history", first)]);
+ const firstKeys = getTraceBatchFirstKeys(0, Date.now()).keys;
+ expect(
+ (await Promise.all(firstKeys.map((key) => client().hlen(key)))).reduce(
+ (sum, count) => sum + count,
+ 0,
+ ),
+ ).toBe(1);
+ await makeDue("project", "history");
+ await runner().processBatch();
+ await vi.waitFor(async () => {
+ expect(await client().exists(...firstKeys)).toBe(0);
+ });
+ const next = first + 300_000;
+ await trackTraceBatchActivity("project", [event("history", next)]);
+ const state = JSON.parse(
+ (await client().hget(stateKey, member("project", "history")))!,
+ );
+ expect(state).toMatchObject({
+ minStart: next,
+ eventUpdateCount: 1,
+ serializedEventBytes: 100,
+ });
+ expect(await client().pttl(dueKey)).toBeGreaterThan(150);
+ });
+
+ it("distributes skewed projects by trace and dispatches every shard with single-slot scripts", async () => {
+ const shardCount = 64;
+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = shardCount;
+ const evaluate = client().eval.bind(client());
+ const slots: string[][] = [];
+ vi.spyOn(client(), "eval").mockImplementation(((
+ script: string,
+ count: number,
+ ...args: (string | number | Buffer)[]
+ ) => {
+ slots.push(args.slice(0, count).map(String));
+ return evaluate(script, count, ...args);
+ }) as typeof evaluate);
+ const evaluateCached = client().evalsha.bind(client());
+ vi.spyOn(client(), "evalsha").mockImplementation(((
+ scriptHash: string,
+ count: number,
+ ...args: (string | number | Buffer)[]
+ ) => {
+ slots.push(args.slice(0, count).map(String));
+ return evaluateCached(scriptHash, count, ...args);
+ }) as typeof evaluateCached);
+ const count = 2_000;
+ for (let project = 0; project < 11; project++) {
+ const size = project === 0 ? 600 : 140;
+ await trackTraceBatchActivity(
+ `project-${project}`,
+ Array.from({ length: size }, (_, i) => event(`trace-${project}-${i}`)),
+ );
+ }
+ const counts = [];
+ for (let shard = 0; shard < shardCount; shard++) {
+ const { due } = getTraceBatchReadinessKeys(shard);
+ const members = await client().zrange(due, 0, -1);
+ counts.push(members.length);
+ for (const entry of members)
+ expect(getTraceBatchShard(entry)).toBe(shard);
+ if (members.length)
+ await client().zadd(
+ due,
+ ...members.flatMap((entry) => [Date.now() - 1_000, entry]),
+ );
+ }
+ expect(counts.reduce((sum, n) => sum + n, 0)).toBe(count);
+ expect(Math.max(...counts)).toBeLessThanOrEqual((1.5 * count) / shardCount);
+ await runner().processBatch();
+ const dispatched = (await queue.getJobs(["wait"])).flatMap(
+ (job) => job.data.payload.traces,
+ );
+ expect(dispatched).toHaveLength(count);
+ expect(
+ new Set(
+ dispatched.map(({ projectId, traceId }) => member(projectId, traceId)),
+ ).size,
+ ).toBe(count);
+ for (const keys of slots) {
+ expect(new Set(keys.map((key) => key.match(/\{[^}]+\}/)?.[0])).size).toBe(
+ 1,
+ );
+ }
+ });
+
+ it.each(["project", "locality"] as const)(
+ "packs one project's traces across readiness shards with %s",
+ async (strategy) => {
+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 16;
+ env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;
+ const traceIds = Array.from(
+ { length: 60 },
+ (_, index) => `cross-shard-${index}`,
+ );
+ await trackTraceBatchActivity(
+ "project",
+ traceIds.map((traceId) => event(traceId)),
+ );
+ for (const traceId of traceIds) {
+ const entry = member("project", traceId);
+ await client().zadd(
+ getTraceBatchReadinessKeys(getTraceBatchShard(entry)).due,
+ Date.now() - 1_000,
+ entry,
+ );
+ }
+ await runner().processBatch();
+ const jobs = await queue.getJobs(["wait"]);
+ expect(jobs).toHaveLength(1);
+ expect(jobs[0].data.payload.traces).toHaveLength(60);
+ for (let shard = 0; shard < 16; shard++) {
+ const { due, state } = getTraceBatchReadinessKeys(shard);
+ expect(await client().zcard(due)).toBe(0);
+ expect(await client().hlen(state)).toBe(0);
+ }
+ },
+ );
+
+ it("acknowledges enqueued batches by shard across a hydration window", async () => {
+ env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 16;
+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 30;
+ const traceIds = Array.from(
+ { length: 120 },
+ (_, index) => `trace-${index}`,
+ );
+ await trackTraceBatchActivity(
+ "project",
+ traceIds.map((traceId) => event(traceId)),
+ );
+ for (const traceId of traceIds) {
+ const entry = member("project", traceId);
+ await client().zadd(
+ getTraceBatchReadinessKeys(getTraceBatchShard(entry)).due,
+ Date.now() - 1_000,
+ entry,
+ );
+ }
+ const evaluate = vi.spyOn(client(), "evalsha");
+ await runner().processBatch();
+ expect(await queue.getWaitingCount()).toBe(4);
+ const acknowledgements = evaluate.mock.calls.filter(
+ ([, keyCount]) => Number(keyCount) > 2,
+ );
+ expect(acknowledgements.length).toBeLessThanOrEqual(16);
+ for (let shard = 0; shard < 16; shard++) {
+ const { due, state } = getTraceBatchReadinessKeys(shard);
+ expect(await client().zcard(due)).toBe(0);
+ expect(await client().hlen(state)).toBe(0);
+ }
+ });
+
it("samples prefixed map memory and paused/delayed queue work, then reports drained zeros", async () => {
const now = Date.now();
const clock = vi.spyOn(Date, "now").mockReturnValue(now);
@@ -645,11 +934,11 @@ describe("trace micro-batch scheduling with Redis", () => {
const evaluate = vi.spyOn(client(), "eval");
await metrics["execute"]();
- for (const key of ["due", "state"]) {
+ for (const key of ["due", "state", "first"]) {
expect(recordGauge).toHaveBeenCalledWith(
"langfuse.trace_batch.redis_key_entries",
1,
- { key, unit: "records" },
+ { key, shard: "total", unit: "records" },
);
const sample = vi
.mocked(recordGauge)
@@ -679,7 +968,7 @@ describe("trace micro-batch scheduling with Redis", () => {
clock.mockReturnValue(now + 30_000);
await metrics["execute"]();
expect(evaluate).toHaveBeenCalledTimes(1);
- await client().del(dueKey, stateKey);
+ await clearReadiness();
await queue.drain(true);
vi.mocked(recordGauge).mockClear();
clock.mockReturnValue(now + 60_000);
@@ -689,7 +978,7 @@ describe("trace micro-batch scheduling with Redis", () => {
"langfuse.trace_batch.active_reads",
0,
);
- expect(vi.mocked(recordGauge).mock.calls).toHaveLength(7);
+ expect(vi.mocked(recordGauge).mock.calls).toHaveLength(15);
expect(
vi.mocked(recordGauge).mock.calls.every(([, value]) => value === 0),
).toBe(true);
@@ -820,6 +1109,63 @@ describe("trace micro-batch scheduling with Redis", () => {
expect.objectContaining({ projectId: "project", traceId, reason }),
);
}
+ await trackTraceBatchActivity("project", [
+ event("many-updates", 1_000_004),
+ event("large-input", 1_000_001),
+ ]);
+ for (const [traceId, eventUpdateCount, serializedEventBytes] of [
+ ["many-updates", 5, 500],
+ ["large-input", 2, 5_100],
+ ] as const)
+ expect(
+ JSON.parse(
+ (await client().hget(stateKey, member("project", traceId)))!,
+ ),
+ ).toMatchObject({ eventUpdateCount, serializedEventBytes });
+ await makeDue("project", "many-updates", "large-input");
+ await runner().processBatch();
+ expect(await queue.getWaitingCount()).toBe(1);
+ expect(await client().zcard(dueKey)).toBe(0);
+ expect(await client().hlen(stateKey)).toBe(0);
+ });
+
+ it("uses retained full-trace estimates for batch budgets after successful dispatch", async () => {
+ const original = env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES;
+ env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES = 6;
+ try {
+ const ingest = async (start: number) => {
+ await trackTraceBatchActivity("project", [
+ event("a", start),
+ event("a", start),
+ event("b", start),
+ event("b", start),
+ ]);
+ await makeDue("project", "a", "b");
+ await runner().processBatch();
+ };
+ await ingest(1_000_000);
+ const [first] = await queue.getJobs(["wait"]);
+ expect(first.data.payload.traces).toHaveLength(2);
+ await ingest(1_000_001);
+ const lateJobs = (await queue.getJobs(["wait"])).filter(
+ (job) => job.id !== first.id,
+ );
+ expect(lateJobs).toHaveLength(2);
+ expect(lateJobs.flatMap((job) => job.data.payload.traces)).toHaveLength(
+ 2,
+ );
+ for (const job of lateJobs) {
+ expect(job.data.payload.traces).toHaveLength(1);
+ expect(job.data.payload.traces[0].minStart).toBe(1_000_000);
+ }
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.estimated_event_update_count",
+ 4,
+ { scope: "trace", strategy: "project" },
+ );
+ } finally {
+ env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES = original;
+ }
});
it("does not report an oversized trace as excluded when new activity keeps it pending", async () => {
@@ -832,13 +1178,18 @@ describe("trace micro-batch scheduling with Redis", () => {
),
);
await makeDue("project", "reactivated");
- const evaluate = client().eval.bind(client());
+ const evaluate = client().evalsha.bind(client());
let reactivated = false;
- vi.spyOn(client(), "eval").mockImplementation((async (
- script: string,
+ vi.spyOn(client(), "evalsha").mockImplementation((async (
+ scriptHash: string,
+ keyCount: number,
...args: (string | number)[]
) => {
- if (!reactivated && script.includes("cjson.decode(state).revision ==")) {
+ if (
+ !reactivated &&
+ keyCount > 2 &&
+ Number(args[keyCount]) === env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS
+ ) {
reactivated = true;
const traceMember = member("project", "reactivated");
const state = JSON.parse((await client().hget(stateKey, traceMember))!);
@@ -848,7 +1199,7 @@ describe("trace micro-batch scheduling with Redis", () => {
JSON.stringify({ ...state, revision: randomUUID() }),
);
}
- return evaluate(script, ...args);
+ return evaluate(scriptHash, keyCount, ...args);
}) as typeof evaluate);
await runner().processBatch();
@@ -1066,6 +1417,42 @@ describe("trace micro-batch scheduling with Redis", () => {
});
it("ingests through BullMQ, postpones readiness on new activity, and consumes full ClickHouse payloads once due", async () => {
+ const orgId = randomUUID();
+ const projectId = randomUUID();
+ await prisma.organization.create({
+ data: {
+ id: orgId,
+ name: "Trace batch late arrivals",
+ projects: { create: { id: projectId, name: "Trace batch" } },
+ },
+ });
+ const usage = {
+ providedUsageDetails: {},
+ usageDetails: {},
+ providedCostDetails: {},
+ costDetails: {},
+ };
+ vi.mocked(summarizeTopicTraceFacets).mockImplementation(
+ async (_models, facets) => ({
+ output: Object.fromEntries(
+ facets.map(({ key }) => [
+ key,
+ {
+ notes: "The complete trace was read.",
+ summary: `summary-${vi.mocked(summarizeTopicTraceFacets).mock.calls.length}`,
+ status: "applicable" as const,
+ },
+ ]),
+ ),
+ ...usage,
+ }),
+ );
+ vi.mocked(embedTopicSummary).mockImplementation(
+ async (_model, _text, dimensions) => ({
+ embedding: Array.from({ length: dimensions }, () => 0.1),
+ ...usage,
+ }),
+ );
const original = {
LANGFUSE_TRACE_BATCH_IDLE_MS: env.LANGFUSE_TRACE_BATCH_IDLE_MS,
LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,
@@ -1110,21 +1497,25 @@ describe("trace micro-batch scheduling with Redis", () => {
batchWorker.waitUntilReady(),
batchEvents.waitUntilReady(),
]);
- const projectId = randomUUID();
// The evaluator sampler assigns these scores 0.0593, 0.0291 and 0.7116.
const traceId = "00000000000000000000000000000001";
const otherTraceId = "00000000000000000000000000000003";
const excludedTraceId = "00000000000000000000000000000002";
- const input = JSON.stringify({ message: "input".repeat(1_000) });
- const output = JSON.stringify({ message: "output".repeat(1_000) });
+ const input = JSON.stringify([
+ { role: "user", content: "input".repeat(1_000) },
+ ]);
+ const output = JSON.stringify({
+ role: "assistant",
+ content: "output".repeat(1_000),
+ });
const metadata = JSON.stringify({ context: "metadata".repeat(1_000) });
- const nano = BigInt(Date.now()) * 1_000_000n;
- const timestamp = {
- low: Number(nano & 0xffffffffn),
- high: Number(nano >> 32n),
- unsigned: true,
- };
- async function ingest(traceIds: string[]) {
+ const nano = BigInt(Date.now() - 20 * 60_000) * 1_000_000n;
+ async function ingest(traceIds: string[], start = nano) {
+ const timestamp = {
+ low: Number(start & 0xffffffffn),
+ high: Number(start >> 32n),
+ unsigned: true,
+ };
// S3 transport and unrelated evaluator configuration are stubbed;
// OTLP conversion, writer, Redis scripts and both queue workers are real.
vi.mocked(getS3EventStorageClient).mockReturnValue({
@@ -1144,7 +1535,7 @@ describe("trace micro-batch scheduling with Redis", () => {
endTimeUnixNano: timestamp,
status: { code: 2, message: "tool call failed" },
attributes: Object.entries({
- "langfuse.observation.type": "span",
+ "langfuse.observation.type": "generation",
"langfuse.observation.input": input,
"langfuse.observation.output": output,
"langfuse.observation.metadata": metadata,
@@ -1234,6 +1625,8 @@ describe("trace micro-batch scheduling with Redis", () => {
excludedObservations++;
}
expect(excludedObservations).toBe(2);
+ topicsProjects.add(projectId);
+ const facets = await ensureDefaultTopicFacets(projectId);
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0;
await waitUntilRedisTime(renewedDue);
@@ -1264,6 +1657,79 @@ describe("trace micro-batch scheduling with Redis", () => {
expect(await client().hlen(stateKey)).toBe(0);
await dispatcher.processBatch();
expect(await queue.getCompletedCount()).toBe(1);
+ expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(2);
+ const summaries = async () =>
+ (
+ await Promise.all(
+ facets.map((facet) =>
+ listTopicSummaries(
+ projectId,
+ {
+ traceIds: [traceId, otherTraceId],
+ facetId: facet.id,
+ facetVersion: facet.versions[0].version,
+ },
+ {
+ from: new Date(Number(nano / 1_000_000n) - 1),
+ to: new Date(),
+ },
+ ),
+ ),
+ )
+ ).flat();
+ const firstSummaries = await summaries();
+ expect(firstSummaries).toHaveLength(8);
+
+ // One late event stays within the original buffer, while the other is
+ // ten minutes away and requires the retained earliest start.
+ env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;
+ await ingest([traceId], nano + 60_000n * 1_000_000n);
+ await ingest([otherTraceId], nano + 10n * 60_000n * 1_000_000n);
+ await writer.flushAll(true);
+ await makeDue(projectId, traceId, otherTraceId);
+ await dispatcher.processBatch();
+ const lateBatch = (
+ await queue.getJobs(["wait", "active", "completed"])
+ ).find((job) => job.id !== batch.id)!;
+ for (const trace of lateBatch.data.payload.traces)
+ expect(trace.minStart).toBe(Number(nano / 1_000_000n));
+ const lateResult = await lateBatch.waitUntilFinished(batchEvents, 10_000);
+ expect(lateResult).toMatchObject({ observationCount: 5, traceCount: 2 });
+ expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(4);
+
+ await ingest([otherTraceId], nano + 3n * 60_000n * 1_000_000n);
+ await writer.flushAll(true);
+ await makeDue(projectId, otherTraceId);
+ await dispatcher.processBatch();
+ const backdatedBatch = (
+ await queue.getJobs(["wait", "active", "completed"])
+ ).find((job) => job.id !== batch.id && job.id !== lateBatch.id)!;
+ expect(backdatedBatch.data.payload.traces).toMatchObject([
+ {
+ minStart: Number(nano / 1_000_000n),
+ maxStart: Number(nano / 1_000_000n) + 10 * 60_000,
+ },
+ ]);
+ expect(
+ await backdatedBatch.waitUntilFinished(batchEvents, 10_000),
+ ).toMatchObject({ observationCount: 3, traceCount: 1 });
+ expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(5);
+ const latest = await summaries();
+ expect(latest).toHaveLength(8);
+ for (const summary of latest) {
+ expect(summary.unitStartTime).toBe(
+ firstSummaries.find((row) => row.traceId === summary.traceId)!
+ .unitStartTime,
+ );
+ expect(summary.summary).not.toBe(
+ firstSummaries.find(
+ (row) =>
+ row.traceId === summary.traceId &&
+ row.facetId === summary.facetId,
+ )!.summary,
+ );
+ }
} finally {
try {
await Promise.all([ingestionWorker.close(), batchWorker.close()]);
@@ -1272,6 +1738,8 @@ describe("trace micro-batch scheduling with Redis", () => {
await ingestionQueue.close();
await writer.shutdown();
} finally {
+ topicsProjects.delete(projectId);
+ await prisma.organization.deleteMany({ where: { id: orgId } });
Object.assign(env, original);
}
}
@@ -1295,6 +1763,16 @@ describe("trace micro-batch scheduling with Redis", () => {
}
});
+ it("does not load idle settings when every trace is sampled out", async () => {
+ env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = Number.MIN_VALUE;
+ vi.mocked(shared.getTracingReadinessSettings).mockClear();
+
+ await trackTraceBatchActivity("project", [event("sampled-out")]);
+
+ expect(shared.getTracingReadinessSettings).not.toHaveBeenCalled();
+ expect(await client().hlen(stateKey)).toBe(0);
+ });
+
it("keeps stable nested trace samples across replay and counts decisions once per trace in each ingestion batch", async () => {
const traceIds = Array.from({ length: 1_000 }, (_, i) =>
i.toString(16).padStart(32, "0"),
@@ -1343,26 +1821,48 @@ describe("trace micro-batch scheduling with Redis", () => {
["eligible", 1_000],
...(expectedCount > 0 ? [["recorded", expectedCount] as const] : []),
] as const) {
- expect(recordIncrement).toHaveBeenCalledWith(
- "langfuse.trace_batch.event_updates",
- count * 3,
- { stage },
- );
- expect(recordIncrement).toHaveBeenCalledWith(
- "langfuse.trace_batch.serialized_event_bytes",
- count * 350,
- { stage },
- );
+ for (const [metric, total] of [
+ ["event_updates", count * 3],
+ ["serialized_event_bytes", count * 350],
+ ] as const)
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(
+ ([name, , tags]) =>
+ name === `langfuse.trace_batch.${metric}` &&
+ tags?.stage === stage,
+ )
+ .reduce((sum, [, value]) => sum + value, 0),
+ ).toBe(total);
}
previous = selected;
}
});
it("counts only acknowledged update volume when tracking fails after a chunk", async () => {
- const evaluate = client().eval.bind(client());
- vi.spyOn(client(), "eval")
- .mockImplementationOnce(evaluate)
- .mockRejectedValueOnce(new Error("tracking connection failed"));
+ await trackTraceBatchActivity("warm-up", [event("trace")]);
+ await clearReadiness();
+ vi.mocked(recordIncrement).mockClear();
+ const fallback = vi.spyOn(client(), "eval");
+ const pipeline = client().pipeline.bind(client());
+ let commands = 0;
+ vi.spyOn(client(), "pipeline").mockImplementation(() => {
+ const batch = pipeline();
+ const evaluate = batch.evalsha.bind(batch);
+ vi.spyOn(batch, "evalsha").mockImplementation(((
+ scriptHash: string,
+ keyCount: number,
+ ...args: (string | number | Buffer)[]
+ ) => {
+ // Fail one command before mutation; the other pipelined chunks still execute.
+ if (++commands === 2) args[1] = args[0];
+ // Simulate a cache miss for one sibling without flushing shared Redis scripts.
+ if (commands === 3) scriptHash = "0".repeat(40);
+ return evaluate(scriptHash, keyCount, ...args);
+ }) as typeof evaluate);
+ return batch;
+ });
await expect(
trackTraceBatchActivity("project", [
@@ -1373,7 +1873,15 @@ describe("trace micro-batch scheduling with Redis", () => {
]),
).resolves.toBeUndefined();
- expect(await client().hlen(stateKey)).toBe(1_000);
+ const recorded = await client().hlen(stateKey);
+ expect(recorded).toBe(150);
+ expect(commands).toBe(4);
+ expect(fallback).toHaveBeenCalledOnce();
+ for (const state of await client().hvals(stateKey))
+ expect(JSON.parse(state)).toMatchObject({
+ eventUpdateCount: 1,
+ serializedEventBytes: 7,
+ });
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.event_updates",
1_001,
@@ -1384,16 +1892,20 @@ describe("trace micro-batch scheduling with Redis", () => {
expect.anything(),
{ stage: "sampled" },
);
- expect(recordIncrement).toHaveBeenCalledWith(
- "langfuse.trace_batch.event_updates",
- 1_000,
- { stage: "recorded" },
- );
- expect(recordIncrement).toHaveBeenCalledWith(
- "langfuse.trace_batch.serialized_event_bytes",
- 7_000,
- { stage: "recorded" },
- );
+ for (const [metric, count] of [
+ ["event_updates", recorded],
+ ["serialized_event_bytes", recorded * 7],
+ ] as const)
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(
+ ([name, , tags]) =>
+ name === `langfuse.trace_batch.${metric}` &&
+ tags?.stage === "recorded",
+ )
+ .reduce((sum, [, value]) => sum + value, 0),
+ ).toBe(count);
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.trace_batch.event_updates",
1,
@@ -1593,7 +2105,7 @@ describe("trace micro-batch scheduling with Redis", () => {
},
);
- it("keeps legacy estimates unknown through updates and restores them after dispatch", async () => {
+ it("keeps unknown estimates through updates and late-arrival dispatch", async () => {
const legacyStates = [
{},
{ eventUpdateCount: 4 },
@@ -1611,6 +2123,11 @@ describe("trace micro-batch scheduling with Redis", () => {
...estimates,
}),
);
+ await client().zadd(
+ dueKey,
+ Date.now() + 600_000,
+ member("project", legacyIds[index]),
+ );
}
await trackTraceBatchActivity("project", [
...legacyIds.map((traceId) => event(traceId, 1_000_001, 20)),
@@ -1667,34 +2184,26 @@ describe("trace micro-batch scheduling with Redis", () => {
legacyIds.map((traceId) => event(traceId, 1_000_002, 20)),
);
for (const traceId of legacyIds) {
- expect(
- JSON.parse(
- (await client().hget(stateKey, member("project", traceId)))!,
- ),
- ).toMatchObject({
- eventUpdateCount: 1,
- serializedEventBytes: 20,
- });
+ const state = JSON.parse(
+ (await client().hget(stateKey, member("project", traceId)))!,
+ );
+ expect(state.eventUpdateCount).toBeUndefined();
+ expect(state.serializedEventBytes).toBeUndefined();
}
await makeDue("project", ...legacyIds);
await dispatcher.processBatch();
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.estimated_event_update_count",
- 3,
- { scope: "batch", strategy: "project" },
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.estimated_serialized_event_bytes",
- 60,
- { scope: "batch", strategy: "project" },
- );
expect(
vi
- .mocked(recordIncrement)
- .mock.calls.some(
- ([name]) => name === "langfuse.trace_batch.estimates_unavailable",
+ .mocked(recordDistribution)
+ .mock.calls.filter(([name]) =>
+ name.startsWith("langfuse.trace_batch.estimated_"),
),
- ).toBe(false);
+ ).toHaveLength(0);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.estimates_unavailable",
+ 1,
+ { scope: "batch", strategy: "project" },
+ );
});
it.each(["project", "locality"] as const)(
@@ -1862,15 +2371,11 @@ describe("trace micro-batch scheduling with Redis", () => {
const hydratedRevision = JSON.parse(
(await client().hget(stateKey, targetMember))!,
).revision;
- const evaluate = client().eval.bind(client());
+ const evaluate = client().evalsha.bind(client());
let reactivated = false;
- vi.spyOn(client(), "eval").mockImplementation(async (...args) => {
+ vi.spyOn(client(), "evalsha").mockImplementation(async (...args) => {
const result = await evaluate(...args);
- if (
- !reactivated &&
- typeof args[0] === "string" &&
- args[0].includes("local result = {}")
- ) {
+ if (!reactivated && Number(args[1]) === 2 && args.length > 6) {
reactivated = true;
await trackTraceBatchActivity("project", [
event(early[0].traceId, 11 * minute),
@@ -1980,12 +2485,12 @@ describe("trace micro-batch scheduling with Redis", () => {
});
it("expires abandoned pending keys natively without another ingestion or dispatcher run", async () => {
- env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;
+ env.LANGFUSE_TRACE_BATCH_IDLE_MS = 100;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 300;
await trackTraceBatchActivity("project", [event("abandoned")]);
for (const key of [dueKey, stateKey]) {
expect(await client().pttl(key)).toBeGreaterThan(0);
- expect(await client().pttl(key)).toBeLessThanOrEqual(300);
+ expect(await client().pttl(key)).toBeLessThanOrEqual(400);
}
await vi.waitFor(async () => {
expect(await client().exists(dueKey, stateKey)).toBe(0);
@@ -1999,7 +2504,7 @@ describe("trace micro-batch scheduling with Redis", () => {
await trackTraceBatchActivity("project", [event("long-idle")]);
const initialDeadline = Number(await client().pexpiretime(dueKey));
await vi.waitFor(() =>
- expect(Date.now()).toBeGreaterThan(initialDeadline - 60_000 + 10),
+ expect(Date.now()).toBeGreaterThan(initialDeadline - 70_000 + 10),
);
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 20_000;
await trackTraceBatchActivity("project", [event("later")]);
@@ -2018,6 +2523,24 @@ describe("trace micro-batch scheduling with Redis", () => {
);
expect(await client().zcard(dueKey)).toBe(3);
expect(await client().hlen(stateKey)).toBe(3);
+
+ const roundedDeadline =
+ Math.ceil((Date.now() + 4 * 60 * 60_000) / 100_000_000) * 100_000_000;
+ for (const key of [dueKey, stateKey])
+ await client().pexpireat(key, roundedDeadline);
+ vi.mocked(recordIncrement).mockClear();
+ await trackTraceBatchActivity("project", [event("rounded-expiry")]);
+ expect(recordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.tracking_errors",
+ expect.anything(),
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.event_updates",
+ 1,
+ { stage: "recorded" },
+ );
+ for (const key of [dueKey, stateKey])
+ expect(Number(await client().pexpiretime(key))).toBe(roundedDeadline);
});
it("drains and removes queued jobs without queries when reads are disabled", async () => {
@@ -2096,8 +2619,10 @@ describe("trace micro-batch scheduling with Redis", () => {
);
await makeDue("project", "recent");
await trackTraceBatchActivity("project", [event("refreshed", 2_000_000)]);
- expect(await client().zcard(dueKey)).toBe(3);
- expect(await client().hlen(stateKey)).toBe(3);
+ const pending = await client().zcard(dueKey);
+ expect(pending).toBeGreaterThan(2);
+ expect(pending).toBeLessThan(expired.length + 2);
+ expect(await client().hlen(stateKey)).toBe(pending);
expect(
await client().hget(stateKey, member("project", "recent")),
).not.toBeNull();
@@ -2106,20 +2631,30 @@ describe("trace micro-batch scheduling with Redis", () => {
).toBeGreaterThan(Date.now());
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.expired_traces",
- 1_000,
+ expired.length + 2 - pending,
{ source: "ingestion" },
);
- await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);
+ await vi.waitFor(async () => {
+ await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);
+ expect(await client().zcard(dueKey)).toBe(2);
+ });
expect((await client().hkeys(stateKey)).sort()).toEqual([
member("project", "recent"),
member("project", "refreshed"),
]);
expect(await client().zcard(dueKey)).toBe(2);
- expect(recordIncrement).toHaveBeenCalledWith(
- "langfuse.trace_batch.expired_traces",
- 1,
- { source: "ingestion" },
+ const expirySamples = vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(
+ ([name, , tags]) =>
+ name === "langfuse.trace_batch.expired_traces" &&
+ tags?.source === "ingestion",
+ )
+ .map(([, count]) => count);
+ expect(expirySamples.reduce((sum, count) => sum + count, 0)).toBe(
+ expired.length,
);
+ expect(expirySamples.every((count) => count < expired.length)).toBe(true);
expect(await queue.getWaitingCount()).toBe(0);
});
@@ -2190,6 +2725,25 @@ describe("trace micro-batch scheduling with Redis", () => {
},
);
+ it("acknowledges successful jobs when a later enqueue fails", async () => {
+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 1;
+ await trackTraceBatchActivity("project", [event("first"), event("second")]);
+ await makeDue("project", "first", "second");
+ const add = queue.add.bind(queue);
+ vi.spyOn(queue, "add")
+ .mockImplementationOnce((...args) => add(...args))
+ .mockRejectedValueOnce(new Error("enqueue unavailable"));
+ const dispatcher = runner();
+ await dispatcher.processBatch();
+ expect(await queue.getWaitingCount()).toBe(1);
+ expect(await client().zcard(dueKey)).toBe(1);
+ expect(await client().hlen(stateKey)).toBe(1);
+ await dispatcher.processBatch();
+ expect(await queue.getWaitingCount()).toBe(2);
+ expect(await client().zcard(dueKey)).toBe(0);
+ expect(await client().hlen(stateKey)).toBe(0);
+ });
+
it.each(["updated", "deleted", "expired"] as const)(
"preserves arrivals during enqueue, including deleted/recreated state: %s",
async (mode) => {
@@ -2222,6 +2776,10 @@ describe("trace micro-batch scheduling with Redis", () => {
const [job] = await queue.getJobs(["wait"]);
expect(current.revision).not.toBe(job.data.payload.traces[0].revision);
expect(current.maxStart).toBe(2_000_000);
+ expect(current).toMatchObject({
+ eventUpdateCount: 2,
+ serializedEventBytes: 200,
+ });
await makeDue("project", "trace");
await dispatcher.processBatch();
expect(await queue.getWaitingCount()).toBe(2);
@@ -2238,7 +2796,7 @@ describe("trace micro-batch scheduling with Redis", () => {
vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
const job = await add(...args);
// The next Redis operation is the post-enqueue acknowledgement.
- vi.spyOn(client(), "eval").mockRejectedValueOnce(
+ vi.spyOn(client(), "evalsha").mockRejectedValueOnce(
new Error("ACK unavailable"),
);
return job;
diff --git a/worker/src/env.ts b/worker/src/env.ts
index ae53a57ae3..3e0fef80fb 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -212,6 +212,14 @@ const EnvSchema = z.object({
.enum(["project", "locality"])
.default("project"),
LANGFUSE_TRACE_BATCH_IDLE_MS: z.coerce.number().int().positive(),
+ LANGFUSE_TRACE_BATCH_SHARD_COUNT: z.coerce.number().int().min(1).max(256),
+ LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .max(2 * 24 * 60 * 60 * 1_000)
+ .default(24 * 60 * 60 * 1_000),
+ // Recovery grace after readiness; whole-key expiry includes the idle interval.
LANGFUSE_TRACE_BATCH_PENDING_TTL_MS: z.coerce
.number()
.int()
@@ -890,6 +898,10 @@ const parseEnv = (): ParsedEnv => {
source.LANGFUSE_TRACE_BATCH_IDLE_MS =
source.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV" ? "120000" : "600000";
}
+ if (source.LANGFUSE_TRACE_BATCH_SHARD_COUNT === undefined) {
+ source.LANGFUSE_TRACE_BATCH_SHARD_COUNT =
+ source.REDIS_CLUSTER_ENABLED === "true" ? "16" : "1";
+ }
const parsed = EnvSchema.parse(source);
validateV4Flags(parsed);
validateInAppAgentSandboxConfig(parsed);
diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.ts
index 9eb6adaa5e..40c6f5804e 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts
@@ -133,11 +133,12 @@ beforeEach(() => {
});
describe("summarizeAssembledTrace", () => {
- it("summarizes the assembled transcript once per current facet and skips a finished retry", async () => {
+ it("skips a finished job retry but summarizes a later redispatch", async () => {
await summarizeAssembledTrace({
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -183,18 +184,16 @@ describe("summarizeAssembledTrace", () => {
expect.any(Number),
{ stage: "trace", outcome: "success", unit: "milliseconds" },
);
- state.stored.mockImplementation(async (_projectId, _filter, timeRange) =>
- new Date(written.unitStartTime) >= timeRange.from &&
- new Date(written.unitStartTime) < timeRange.to
- ? [written]
- : [],
- );
+ state.stored.mockResolvedValue([
+ { ...written, processedAt: "2026-09-22T12:02:00.000Z" },
+ ]);
state.summarize.mockClear();
state.write.mockClear();
await summarizeAssembledTrace({
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -206,7 +205,8 @@ describe("summarizeAssembledTrace", () => {
await summarizeAssembledTrace({
projectId: "project-a",
traceId: "trace-1",
- traceTimestamp: "2026-09-22T11:59:00.000Z",
+ traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:03:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -214,9 +214,11 @@ describe("summarizeAssembledTrace", () => {
facets: state.facets(),
});
expect(state.summarize).toHaveBeenCalledOnce();
+ expect(state.write).toHaveBeenCalledOnce();
expect(state.write.mock.calls[0][0][0].unitStartTime).toBe(
- "2026-09-22T11:59:00.000Z",
+ written.unitStartTime,
);
+ expect(state.stored).toHaveBeenCalledTimes(3);
});
it("summarizes all pending facets in one call and records its usage once", async () => {
@@ -238,6 +240,7 @@ describe("summarizeAssembledTrace", () => {
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -266,6 +269,7 @@ describe("summarizeAssembledTrace", () => {
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -298,6 +302,7 @@ describe("summarizeAssembledTrace", () => {
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -355,6 +360,7 @@ describe("summarizeAssembledTrace", () => {
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -391,6 +397,7 @@ describe("summarizeAssembledTrace", () => {
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -433,6 +440,7 @@ describe("summarizeAssembledTrace", () => {
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
@@ -470,6 +478,7 @@ describe("summarizeAssembledTrace", () => {
projectId: "project-a",
traceId: "trace-1",
traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
environment: "default",
traceName: "agent-turn",
transcript,
diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.ts
index ca63f351c6..79b7646957 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.ts
@@ -40,6 +40,7 @@ type AssembledTraceInput = {
projectId: string;
traceId: string;
traceTimestamp: string;
+ jobEnqueuedAt: Date;
environment: string;
traceName: string;
transcript: Transcript | null;
@@ -49,7 +50,7 @@ type AssembledTraceInput = {
/**
* Summarizes one trace from the transcript the batch job already assembled,
- * with one model call for all of its pending facets. Does not load
+ * with one model call for all of its configured facets. Does not load
* observations again.
*/
export async function summarizeAssembledTrace(
@@ -86,11 +87,6 @@ async function summarizeEnabledTrace(
const config = topicProcessingConfigSchema.parse({
summaryModel: models.summary.model,
});
- const timestamp = Date.parse(input.traceTimestamp);
- const timeRange = {
- from: new Date(timestamp),
- to: new Date(timestamp + 1),
- };
const pending: {
name: string;
version: TopicFacetVersion;
@@ -101,6 +97,11 @@ async function summarizeEnabledTrace(
...facets.filter((facet) => facet.isBuiltIn),
...facets.filter((facet) => !facet.isBuiltIn),
];
+ const timestamp = Date.parse(input.traceTimestamp);
+ const timeRange = {
+ from: new Date(timestamp),
+ to: new Date(timestamp + 1),
+ };
for (const facet of ordered) {
const version = facet.versions[0];
if (facet.projectId !== input.projectId || !version) continue;
@@ -113,8 +114,14 @@ async function summarizeEnabledTrace(
},
timeRange,
);
- if (!stored.length)
- pending.push({ name: facet.name, version, builtIn: facet.isBuiltIn });
+ if (
+ stored.some(
+ (summary) =>
+ Date.parse(summary.processedAt) >= input.jobEnqueuedAt.getTime(),
+ )
+ )
+ continue;
+ pending.push({ name: facet.name, version, builtIn: facet.isBuiltIn });
}
if (!pending.length) return "unchanged";
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index cb56fef35f..95662d8a69 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -7,31 +7,54 @@ flags and transcript-metrics switch below default to `false`; an ordinary releas
needs no infrastructure changes. The four enablement flags stay commented in env
templates. The local dev template sets the idle time below.
+## Readiness requirements
+
+- Every admitted event refreshes the trace's idle deadline. Tracking uses the
+ project's configured idle interval, or the deployment default when unset.
+- Keep pending scheduling state separate from retained historical bounds. Enqueue
+ acknowledgement clears pending state; history survives to support later arrivals.
+- A late arrival within history retention opens a new pending window and reads the
+ whole known trace, using the earliest and latest retained event starts plus the
+ existing two-minute buffer. Backdated arrivals must preserve the previous tail.
+- The consumer checks existing summaries against the job enqueue time. Retries
+ skip facets completed by this job; later redispatches append updated summaries.
+- Bounds, counters, revision and due time are merged atomically. Readiness is
+ revalidated before enqueue, and acknowledgement must preserve any newer revision.
+- Pending state must survive its idle interval plus recovery grace. Historical
+ values have an independent logical TTL and expire without field scans.
+- Use Redis 7-compatible operations on standalone Redis, Sentinel and Cluster.
+ Every multi-key Lua call stays in one slot; cleanup and Lua batches are bounded.
+ One renewable dispatcher lease coordinates scheduling across all shards.
+- Redis loss and events outside retained history can leave gaps. There is no
+ reconciler, and retained timestamps cannot recover data that was never observed.
+
## Controls
-| Variable | Default | Role |
-| --- | --- | --- |
-| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
-| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
-| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
-| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
-| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |
-| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `0` | Fraction of other projects admitted by ingestion, from 0 to 1. Unset or `0` is off. Topics-enabled projects are always admitted. |
-| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
-| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |
-| `LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES` | `1500` | Estimated event updates per multi-trace job; `0` disables. |
-| `LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES` | `67108864` | Estimated serialized event bytes per multi-trace job (64 MiB); `0` disables. |
-| `LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` | `5000` | Exclude a trace whose pending window exceeds this many event updates; `0` disables. |
-| `LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` | `52428800` | Exclude a trace whose pending window exceeds these serialized bytes (50 MiB); `0` disables. |
-| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |
-| `LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS` | `40000` | ClickHouse client socket idle timeout in milliseconds, including pauses for summary backpressure. Independent of the 30-second server query execution limit. |
-| `LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES` | `104857600` | Queued input/output/metadata payload threshold (100 MiB). Includes the active summary; excludes the current trace, tool fields, transport buffers and JavaScript object overhead. |
-| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
-| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
-| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Default idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. A project's `tracing_readiness_settings.trace_idle_ms` (60 s to 60 min) overrides it, capped at the pending TTL minus the dispatch interval. The project value is cached in Redis for 5 minutes; saving replaces it, so the next activity reads the new value. A failed read uses the default and is not cached. |
-| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
-| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
+| Variable | Default | Role |
+| ------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
+| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |
+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `0` | Fraction of other projects admitted by ingestion, from 0 to 1. Unset or `0` is off. Topics-enabled projects are always admitted. |
+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |
+| `LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES` | `1500` | Estimated event updates per multi-trace job; `0` disables. |
+| `LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES` | `67108864` | Estimated serialized event bytes per multi-trace job (64 MiB); `0` disables. |
+| `LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` | `5000` | Exclude a trace whose retained read-window estimate exceeds this many event updates; `0` disables. |
+| `LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` | `52428800` | Exclude a trace whose retained read-window estimate exceeds these serialized bytes (50 MiB); `0` disables. |
+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |
+| `LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS` | `40000` | ClickHouse client socket idle timeout in milliseconds, including pauses for summary backpressure. Independent of the 30-second server query execution limit. |
+| `LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES` | `104857600` | Queued input/output/metadata payload threshold (100 MiB). Includes the active summary; excludes the current trace, tool fields, transport buffers and JavaScript object overhead. |
+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Default idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. A project's `tracing_readiness_settings.trace_idle_ms` (60 seconds to 60 minutes) overrides it, capped at the pending TTL minus the dispatch interval. The project value is cached in Redis for 5 minutes; saving replaces it, and a failed read uses the default without caching. |
+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Recovery grace after a trace becomes due (2 hours). Pending keys expire no earlier than the latest required due-plus-grace deadline. |
+| `LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS` | `86400000` | Historical event-start bounds and size estimates retained after activity or dispatch (24 hours; at most 48 hours). Internal and cloud-gated. |
+| `LANGFUSE_TRACE_BATCH_SHARD_COUNT` | `auto` | 16 on Redis Cluster; 1 on standalone/Sentinel. Override with 1–256. Fixed across Redis resizing; drain before changing. |
+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
Producer, dispatcher and consumer can run independently. Sampling reuses the
evaluator's deterministic trace-ID sampler; every observation of the same trace
@@ -41,42 +64,173 @@ processing, and the shared queue accessor, including admin queue inspection.
## Redis state and expiry
-- `{trace-batch}:due`: sorted set, with JSON `[projectId, traceId]` members and
+- `{trace-batch:<shard>}:due`: sorted set, with JSON `[projectId, traceId]` members and
Redis server time plus idle delay as the score.
-- `{trace-batch}:state`: hash with the same members and JSON
- `{minStart, maxStart, revision, eventUpdateCount, serializedEventBytes}` values. Bounds describe observed event start
+- `{trace-batch:<shard>}:state`: hash with the same members and JSON
+ `{minStart, maxStart, revision, historyBucket, eventUpdateCount, serializedEventBytes}` values. Bounds describe observed event start
times; revision protects newer ingestion from an older dispatch acknowledgment.
- `{trace-batch}:dispatcher`: renewable dispatcher lease.
+- `{trace-batch:<shard>}:first:<bucket>`: hourly hashes retaining the minimum and
+ maximum event starts independently of pending state. Fields are the first 16 binary
+ bytes of SHA-256 of the JSON project/trace pair; values are MessagePack
+ `[minStart, maxStart, expiresAt, eventUpdateCount, serializedEventBytes]`.
+ A bounds-only tuple has unknown estimates. This accepts the negligible collision probability of a
+ 128-bit identity. Key shape is independent of the processing unit.
+
+Sharding uses the shared SHA-256 shard helper on the complete project/trace pair,
+so a large project spreads across shards. Every Lua call uses only keys with
+one shard's hash tag. The single dispatcher lease covers all shards; batch
+packing combines ready traces from every shard. Queue slots are unchanged.
+
+When pending state is absent or has no bucket pointer, tracking looks up the
+bounded set of live time buckets (26 keys for the default TTL), never scans their
+fields, and atomically merges the historical bounds. Valid pending state contains
+both bounds, cumulative size estimates and its current history bucket, allowing direct movement to the new
+bucket on subsequent activity and guarded acknowledgement. The Redis server clock owns
+both retention and readiness. Values with an elapsed deadline are ignored;
+whole-key expiry at bucket end plus retention removes storage without scans.
+Physical removal can lag logical expiry by up to one hour (or the TTL for a
+shorter test override). Activity and guarded acknowledgement refresh retention,
+so the post-dispatch window lasts the configured TTL independently of idle delay.
+One live copy per identity
+is retained under a consistent TTL.
No event payloads are stored here. Atomic Lua updates accumulate estimates and
-refresh bounds, revision and readiness. Each admitted intake
-sets a shared absolute expiry on both pending keys with `PEXPIREAT`, preserving
-any later existing expiry using `PEXPIRETIME` (Redis 7+). With no further intake,
-both keys expire after the retention period even if every application stops;
-idle time is not added to this deadline.
+refresh bounds, revision and readiness:
+
+```text
+due = last admitted activity + effective project idle
+pending expiry = due + recovery grace
+```
-This is a whole-key inactivity backstop, not native per-member expiry. During
-continuous intake, bounded Lua cleanup removes members whose due time is older
-than retention; ingestion and dispatcher both perform cleanup.
+Tracking uses the project's configured idle duration when available. The default
+is 600 seconds outside DEV and 120 seconds in DEV, with the existing environment
+override.
+
+Both pending keys receive a whole-key `PEXPIREAT` covering the required expiry,
+preserving any existing later deadline with `PEXPIRETIME` (Redis 7+). During
+activity, cleanup removes at most 100 members per call with
+`due <= now - recovery grace`, using the existing due index. This supports
+different project idle durations without another per-member expiry index.
+Cleanup runs during ingestion and dispatch; whole-key expiry also clears abandoned
+state if all processes stop. Acknowledgement normally removes pending entries
+as soon as enqueueing succeeds, independently of their safety deadline.
+
+### Sharding and atomicity
+
+The shard is `SHA256(JSON.stringify([projectId, traceId]))[first 32 bits] % N`.
+It is stable for a fixed shard count and hashes the trace identity rather than
+the project alone. Redis Cluster then maps the shared `{trace-batch:N}` tag to
+one of its 16,384 slots; the number of logical readiness shards is independent
+of the number of Redis primaries. Standalone Redis executes the same layout.
+Use a `REDIS_KEY_PREFIX` without its own hash tag: Redis uses the first tag in
+the full key, so a tagged prefix would collapse every shard onto one slot.
+
+```mermaid
+flowchart LR
+ identity[Project + trace identity] --> hash[SHA-256 modulo shard count]
+ hash --> slot[One Redis Cluster slot per shard tag]
+ slot --> due[Due sorted set]
+ slot --> state[Pending state hash]
+ slot --> first[First-seen bucket hashes]
+```
+
+| Operation | Atomic boundary |
+| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| Track activity | One Lua call reads retained starts and pending state, merges min/max and counters, replaces the revision, sets readiness, moves first-seen history and sets expiry. All keys share a slot. |
+| Snapshot/expiry | One Lua call removes a bounded expired cohort and returns readiness counts and the cutoff. |
+| Hydrate | One Lua call rechecks due scores and reads corresponding state/revisions. Freshly reactivated traces are not dispatched from an obsolete snapshot. |
+| Acknowledge | One Lua call compares the revision, refreshes first-seen retention from dispatch time and removes due/state only when it still matches. |
+| Enqueue | A separate BullMQ operation precedes acknowledgement. Stable job IDs reduce duplicates if acknowledgement fails. Concurrent arrivals survive the guarded acknowledgement. |
+
+No application-side read-then-write decides a trace's bounds, revision or due
+score. `TIME` before tracking selects the bounded bucket keys; Lua reads Redis
+time again and rejects a stale bucket selection before mutating anything.
+Scripts serialize within a Redis primary; different primaries can execute
+different shards concurrently. Tracking, hydration and acknowledgement calls
+contain at most 50 traces; cleanup handles at most 100. Ingestion sends at most
+four tracking scripts per pipeline, all in one shard, using one Redis clock lookup
+per ingestion call. A group requiring only one script uses a direct call.
+Successful pipeline commands remain committed when a sibling
+fails; only commands rejected with `NOSCRIPT` are retried using `EVAL`. A pipeline
+is not one atomic operation; each Lua call is atomic independently.
+Dispatch still pools
+up to 1,000 hydrated candidates for locality selection, so script call size does
+not constrain queue batch packing. Scripts use cached `EVALSHA`, falling back
+to `EVAL` after a missing script cache on the selected primary. Dispatch holds one renewable lease across
+all shards, and checks/extends it while making progress. The lease excludes other
+dispatchers; it does not lock ingestion. Only each synchronous Redis command or
+Lua call blocks its primary, not the dispatcher's entire read/pack/enqueue run.
+
+Lua atomicity prevents command interleaving, but is not an ACID transaction
+across Redis, BullMQ and ClickHouse and does not roll back writes on a runtime
+script error. Duplicate runs and the accepted Redis failover gap remain part of
+this pipeline's delivery contract. There is no reconciler.
## Dispatch and reads
-One dispatcher holds the renewable lease. It snapshots ready IDs, sorts them by
-project, then hydrates and revalidates state in chunks of 1,000. Jobs contain up
+One dispatcher holds the renewable lease and snapshots each shard in turn. It
+selects the complete due cohort at each shard's cutoff; there is no per-run trace
+limit. Each due-ID range read and the combined in-memory candidate list grow
+with the due cohort, even though subsequent Lua calls and selection pools are
+bounded. It sorts ready IDs by project and due time across shards, then hydrates
+and revalidates state in single-slot calls of up to 50. Locality
+selection combines those calls into pools of up to 1,000. Jobs contain up
to the configured trace cap and weight budgets; project leftovers can share a job. The dispatcher
enqueues before acknowledging matching revisions, so concurrent intake stays
-pending and failed enqueueing can be retried. Stable job IDs limit duplicates.
-
-### Pending-window size estimates
+pending and failed enqueueing can be retried. Successful enqueues are
+acknowledged by shard after each hydration window, including when a later
+enqueue fails, so Redis calls stay bounded by 50 traces rather than job size.
+Stable job IDs limit duplicates.
+
+Acknowledgement removes due/state only. Late arrivals within first-seen
+retention reopen readiness with the merged historical and incoming minimum and
+maximum starts; the existing two-minute query padding reads the trace again.
+Tracking does not classify events as late by comparing timestamps. Every admitted
+event updates readiness; retained history supplies the earlier bound even when
+the incoming start equals it. History without pending state can also result from
+pending expiry, so it does not prove a previous dispatch. Concurrent arrivals
+replace the revision and survive an acknowledgement of an older snapshot.
+
+The consumer reads the supplied trace window and checks existing summaries for
+each facet. A summary processed since this job was enqueued is skipped on retry;
+a later redispatch generates and appends an updated row. There are no Redis
+completion markers or staged results. BullMQ still uses Redis for queue delivery.
+
+`ReplacingMergeTree(processed_at)` retains the greatest processing timestamp
+when the Topics sort key matches. That key includes the start minute: an event
+revealing a genuinely earlier start in a different minute can leave the previous
+summary under its original key. `processed_at` is assigned when summary
+processing starts, so it orders processing attempts rather than dispatch windows.
+An older, narrower dispatch processed later can replace a summary that includes
+newer events. This is an accepted limitation; dispatch-order versioning must be
+revisited before requiring the newest dispatch window to win.
+
+Retaining the historical maximum preserves the previous tail even when a late
+event's start precedes that tail by more than the query buffer. Once historical
+bounds expire or are lost, the prior window cannot be reconstructed from this map.
+
+### Trace-window size estimates
Accepted direct-v4 writes contribute their final serialized UTF-8 event size after
field overflow handling, reusing the writer's existing `event_bytes` calculation.
The accounting field itself is excluded. Acceptance means admission to the writer,
not confirmation of a ClickHouse flush. `eventUpdateCount` counts accepted updates,
-including retries and repeated observation versions; neither counter represents
-unique observations or a trace's lifetime total. Deletion after enqueue, readiness
-and two-hour retention are unchanged. Later arrivals start another pending window.
-Entries missing either counter remain unknown until dispatch or expiry removes them.
+including retries and repeated observation versions, rather than unique observations.
+
+Pending state and history carry cumulative estimates for the retained read window.
+Tracking adds newly accepted updates to valid pending totals. When reopening a
+trace, it initializes totals from retained history before adding the new updates.
+Acknowledgement preserves estimates with the bounds, including for oversized
+exclusions. A late window therefore cannot make a known oversized trace appear
+small or understate its batch weight by counting only its new events.
+
+Missing estimates remain unknown through updates and redispatch until their
+retained window expires. They are not replaced with zero or a count of only the
+new events. These are admission estimates, not hard limits on actual ClickHouse
+rows or query memory; the two-minute query buffer can include additional rows.
+Tracking-volume metrics still count only updates accepted in the current ingestion
+batch; dispatcher size distributions use the cumulative estimates.
The dispatcher emits `estimated_event_update_count` and
`estimated_serialized_event_bytes` distributions under `langfuse.trace_batch`, tagged
@@ -104,7 +258,7 @@ traces around it into extra jobs. Unknown estimates weigh nothing.
Budget-limited jobs report `fill:partial`; the `scope:batch` estimate
distributions show how close jobs run to the budget.
-After hydration, a trace whose pending-window estimates exceed
+After hydration, a trace whose retained read-window estimates exceed
`LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` or
`LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` is never queued or read. The
dispatcher acknowledges it like a dispatched trace, increments
@@ -113,7 +267,8 @@ dispatcher acknowledges it like a dispatched trace, increments
`Trace batch excluded oversized trace` at warn level with `projectId`, `traceId`,
`reason` and both estimates. Group that log by `@projectId` to find repeat
offenders. Activity after the acknowledgement starts a new pending window that
-is evaluated again, so a trace that keeps growing may be logged more than once.
+is evaluated again using its retained cumulative estimates, so a trace that keeps
+growing may be logged more than once.
Traces with unknown estimates are never excluded.
The worker streams full `events_full` payloads and records batch observation,
@@ -377,15 +532,15 @@ The following metrics use the `langfuse.trace_batch` prefix. All additions are
inside the existing cloud experiment paths; normal ingestion with tracking
disabled performs no new aggregation or Redis calls.
-| Metric | Kind / tags | Meaning |
-| ---------------------------------------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|recorded` | Accepted, valid-start-time updates before sampling, and in Redis-acknowledged chunks. Repeated updates count again. Selected volume is `sampling_decisions`. |
-| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |
-| `failed_read_observation_count`, `failed_read_input_bytes` | Distributions | Partial logical rows and input bytes consumed before a stream failure. Separate from successful throughput. Output and metadata bytes stay on the completion log. |
-| `queue_depth` | Gauge; `type:delayed` | Delayed BullMQ jobs. Waiting, active, and failed depth stay on `langfuse.queue.trace_batch.depth`. Request rate, wait time, and processing time use the generic `langfuse.queue.trace_batch` series, the same as every other queue. Processing time is recorded when the processor returns. Discard reasons stay on `discarded_jobs`. |
-| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |
-| `redis_key_bytes` | Gauge; `key:due\|state` | Estimated memory of each readiness key, including Redis overhead. |
-| `redis_key_entries` | Gauge; `key:due\|state` | `ZCARD` / `HLEN` at the memory snapshot. |
+| Metric | Kind / tags | Meaning |
+| ---------------------------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|recorded` | Accepted, valid-start-time updates before sampling, and in Redis-acknowledged chunks. Repeated updates count again. Selected volume is `sampling_decisions`. |
+| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |
+| `failed_read_observation_count`, `failed_read_input_bytes` | Distributions | Partial logical rows and input bytes consumed before a stream failure. Separate from successful throughput. Output and metadata bytes stay on the completion log. |
+| `queue_depth` | Gauge; `type:delayed` | Delayed BullMQ jobs. Waiting, active, and failed depth stay on `langfuse.queue.trace_batch.depth`. Request rate, wait time, and processing time use the generic `langfuse.queue.trace_batch` series, the same as every other queue. Processing time is recorded when the processor returns. Discard reasons stay on `discarded_jobs`. |
+| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |
+| `redis_key_bytes` | Gauge; `key:due\|state\|first`, `shard:N\|total` | Estimated memory of each readiness key, including Redis overhead. |
+| `redis_key_entries` | Gauge; `key:due\|state\|first`, `shard:N\|total` | `ZCARD` / `HLEN` at the memory snapshot. |
The eligible and recorded counters reuse the already-computed serialized size and
per-trace aggregates. Recorded volume is emitted after each acknowledged Lua
@@ -404,14 +559,17 @@ One failed collection does not suppress the others; failed samples are not
reported as zero. Check `langfuse.periodic_runner.completed` and
`last_healthy_timestamp_seconds` with `runner:trace_batch_metrics` for freshness.
-Memory collection uses one bounded two-key Lua call with `MEMORY USAGE ...
+Memory collection uses bounded per-shard Lua calls with `MEMORY USAGE ...
SAMPLES 5`, preserving the shared key prefix and Redis Cluster slot routing.
Missing keys report zero; there is no keyspace scan or exact full-hash traversal.
Memory sampling estimates storage, not process RSS or event payload size.
Queue/map snapshots are global and can be emitted by several workers: use
max/latest across reporters, never sum duplicates. For map totals, first
-deduplicate reporters per `key`, then add the due/state estimates. In contrast,
+deduplicate reporters per `key,shard`, then add shard estimates, or use the
+`shard:total` series directly. Never add total and shard series together. First-seen
+counts include physically retained expired entries until their bucket expires.
+In contrast,
`active_reads` is local to a worker; distinct live worker series may be summed.
It is emitted on stream start/finish and on each metrics-runner cycle, including
zero when idle, so unchanged long-running reads remain visible.
@@ -494,7 +652,7 @@ querying, including failed attempts. The same ID enters ClickHouse
unset to suppress these per-attempt logs. Strategy attribution uses the
dispatcher metrics described below. Compare
query memory/latency and worker memory as well as input/output/metadata byte
-metrics. Redis storage load tests do not measure selector or query memory.
+metrics. Redis storage metrics do not account for selector or query memory.
### Opt-in locality grouping
@@ -554,6 +712,19 @@ and available instance tags in Datadog before relying on these signals.
## Stop and drain
+Upgrading from the unsharded layout requires a drain; mixed writers and
+dispatchers must not run against different layouts. Disable producers, let the
+old dispatcher empty `{trace-batch}:due` and `{trace-batch}:state`, and drain the
+queue with the old consumers. Stop old dispatchers before deploying all new
+producers/dispatchers. Resume tracking only once all use the new layout. There
+is no retained first-seen history for traces dispatched before this upgrade.
+The same procedure applies before changing shard count or retention. Do not
+delete pending keys to simulate a drain. Old queue payloads remain parseable.
+Draining pending work does not migrate first-seen history. A shard-count change
+must also wait for the old first-seen buckets to expire (TTL plus up to one
+bucket), or explicitly accept losing that retained history. Offline migration
+of first-seen entries is not implemented here.
+
Disable ingestion and dispatch to stop new work. Leave the consumer enabled and
set the read flag to `false` to complete and remove jobs without querying or
parsing their payloads. Jobs at least two hours old are also discarded on pickup.
diff --git a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts
index afa70fa532..b6a1406524 100644
--- a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts
+++ b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts
@@ -7,18 +7,31 @@ import {
import { env } from "../../env";
import { recordTraceBatchActiveReads } from "../../queues/traceBatchQueue";
import { PeriodicRunner } from "../../utils/PeriodicRunner";
+import {
+ getTraceBatchFirstKeys,
+ getTraceBatchReadinessKeys,
+} from "./traceBatchReadiness";
const METRIC_PREFIX = "langfuse.trace_batch";
const MEMORY_INTERVAL_MS = 60_000;
-// EVAL applies the shared client's key prefix and routes both keys to their
+// EVAL applies the shared client's key prefix and routes keys to their
// cluster slot. Raw MEMORY commands do not identify keys in ioredis metadata.
const MEMORY_SCRIPT = `
-return {
+local result = {
redis.call('MEMORY', 'USAGE', KEYS[1], 'SAMPLES', 5) or 0,
redis.call('ZCARD', KEYS[1]),
redis.call('MEMORY', 'USAGE', KEYS[2], 'SAMPLES', 5) or 0,
redis.call('HLEN', KEYS[2])
}
+local bytes = 0
+local entries = 0
+for i = 3, #KEYS do
+ bytes = bytes + (redis.call('MEMORY', 'USAGE', KEYS[i], 'SAMPLES', 5) or 0)
+ entries = entries + redis.call('HLEN', KEYS[i])
+end
+table.insert(result, bytes)
+table.insert(result, entries)
+return result
`;
export class TraceBatchMetricsRunner extends PeriodicRunner {
@@ -109,24 +122,49 @@ export class TraceBatchMetricsRunner extends PeriodicRunner {
private async collectMemory(): Promise<void> {
if (!redis) throw new Error("Trace batch metrics require Redis");
- const [dueBytes, dueEntries, stateBytes, stateEntries] = (await redis.eval(
- MEMORY_SCRIPT,
- 2,
- "{trace-batch}:due",
- "{trace-batch}:state",
- )) as number[];
+ const clock = await redis.time();
+ const now = Number(clock[0]) * 1_000 + Math.floor(Number(clock[1]) / 1_000);
+ const totals = [0, 0, 0, 0, 0, 0];
+ for (let shard = 0; shard < env.LANGFUSE_TRACE_BATCH_SHARD_COUNT; shard++) {
+ const { due, state } = getTraceBatchReadinessKeys(shard);
+ const first = getTraceBatchFirstKeys(shard, now);
+ const values = (await redis.eval(
+ MEMORY_SCRIPT,
+ 2 + first.keys.length,
+ due,
+ state,
+ ...first.keys,
+ )) as number[];
+ this.recordMapMemory(values, String(shard));
+ values.forEach((value, index) => (totals[index] += Number(value)));
+ }
+ this.recordMapMemory(totals, "total");
+ }
+
+ private recordMapMemory(values: number[], shard: string): void {
+ const [
+ dueBytes,
+ dueEntries,
+ stateBytes,
+ stateEntries,
+ firstBytes,
+ firstEntries,
+ ] = values;
// These are global snapshots emitted by each eligible worker. Aggregate
// duplicate reporters with max/latest, never sum their identical gauges.
for (const { key, bytes, entries } of [
{ key: "due", bytes: dueBytes, entries: dueEntries },
{ key: "state", bytes: stateBytes, entries: stateEntries },
+ { key: "first", bytes: firstBytes, entries: firstEntries },
]) {
recordGauge(`${METRIC_PREFIX}.redis_key_bytes`, bytes, {
key,
+ shard,
unit: "bytes",
});
recordGauge(`${METRIC_PREFIX}.redis_key_entries`, entries, {
key,
+ shard,
unit: "records",
});
}
diff --git a/worker/src/features/traceBatching/traceBatchReadiness.ts b/worker/src/features/traceBatching/traceBatchReadiness.ts
new file mode 100644
index 0000000000..2ee136c0a2
--- /dev/null
+++ b/worker/src/features/traceBatching/traceBatchReadiness.ts
@@ -0,0 +1,35 @@
+import { createHash } from "node:crypto";
+import { getShardIndex } from "@langfuse/shared/src/server";
+import { env } from "../../env";
+
+export function getTraceBatchReadinessKeys(shard: number) {
+ const tag = `{trace-batch:${shard}}`;
+ return { due: `${tag}:due`, state: `${tag}:state` };
+}
+
+export function getTraceBatchShard(member: string): number {
+ return getShardIndex(member, env.LANGFUSE_TRACE_BATCH_SHARD_COUNT);
+}
+
+// Redis owns the clock. Include the next bucket so a boundary crossed between
+// TIME and EVAL still writes into the correct bucket without a retry.
+export function getTraceBatchFirstKeys(shard: number, now: number) {
+ const ttl = env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS;
+ const width = Math.min(3_600_000, ttl);
+ const firstBucket = Math.floor((now - ttl) / width);
+ const lastBucket = Math.floor(now / width) + 1;
+ return {
+ width,
+ firstBucket,
+ keys: Array.from(
+ { length: lastBucket - firstBucket + 1 },
+ (_, index) => `{trace-batch:${shard}}:first:${firstBucket + index}`,
+ ),
+ };
+}
+
+// A fixed 128-bit identity keeps arbitrary project/trace identifiers out of the
+// retained map. Due/state keep the full identifiers for dispatch.
+export function getTraceBatchFirstMember(member: string): Buffer {
+ return createHash("sha256").update(member).digest().subarray(0, 16);
+}
diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.ts
index afce38c302..7961916c7d 100644
--- a/worker/src/features/traceBatching/traceBatching.ts
+++ b/worker/src/features/traceBatching/traceBatching.ts
@@ -16,16 +16,21 @@ import {
import { isTopicsProjectEnabled } from "@langfuse/shared/topics/server";
import { env } from "../../env";
import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";
+import {
+ getTraceBatchFirstKeys,
+ getTraceBatchFirstMember,
+ getTraceBatchReadinessKeys,
+ getTraceBatchShard,
+} from "./traceBatchReadiness";
import {
getDeterministicSamplingValue,
shouldSampleEvaluation,
} from "../evaluation/deterministicSampling";
-// The shared client applies REDIS_KEY_PREFIX. Both keys must occupy one
-// Redis Cluster slot for the atomic update, snapshot, and acknowledgement.
-const DUE_KEY = "{trace-batch}:due";
-const STATE_KEY = "{trace-batch}:state";
const CHUNK_SIZE = 1_000;
+const REDIS_CHUNK_SIZE = 50;
+const REDIS_PIPELINE_SIZE = 4;
+const CLEANUP_SIZE = 100;
const QUERY_BUFFER_MS = 2 * 60_000;
const MINUTE_MS = 60_000;
const NARROW_BATCH_MAX_ENVELOPE_MS = 60 * MINUTE_MS;
@@ -579,18 +584,62 @@ const TRACK_SCRIPT = `
${EXPIRE_PENDING_SCRIPT}
local clock = redis.call('TIME')
local now = clock[1] * 1000 + math.floor(clock[2] / 1000)
- for i = 3, #ARGV, 6 do
+ local firstTtl = tonumber(ARGV[3])
+ local width = tonumber(ARGV[4])
+ local bucket = math.floor(now / width)
+ local current = bucket - tonumber(ARGV[5]) + 3
+ if current < 3 or current > #KEYS then
+ return redis.error_reply('Trace readiness bucket clock changed; retry intake')
+ end
+ for i = 6, #ARGV, 7 do
local member = ARGV[i]
local minStart = tonumber(ARGV[i + 1])
local maxStart = tonumber(ARGV[i + 2])
local eventUpdateCount = tonumber(ARGV[i + 4])
local serializedEventBytes = tonumber(ARGV[i + 5])
+ local identity = ARGV[i + 6]
local previous = redis.call('HGET', KEYS[2], member)
- if previous then
- local state = cjson.decode(previous)
+ local state = previous and cjson.decode(previous)
+ local priorDue = tonumber(redis.call('ZSCORE', KEYS[1], member))
+ if not priorDue or priorDue <= now - tonumber(ARGV[2]) then state = nil end
+ if state and state.historyBucket then
+ local old = state.historyBucket - tonumber(ARGV[5]) + 3
+ if old >= 3 and old <= #KEYS and old ~= current then
+ redis.call('HDEL', KEYS[old], identity)
+ end
+ else
+ local retainedUpdates = 0
+ local retainedBytes = 0
+ local estimatesKnown = true
+ for key = 3, #KEYS do
+ local retained = redis.call('HGET', KEYS[key], identity)
+ if retained then
+ local bounds = cmsgpack.unpack(retained)
+ if bounds[3] > now then
+ minStart = math.min(minStart, bounds[1])
+ maxStart = math.max(maxStart, bounds[2])
+ if bounds[4] and bounds[5] then
+ retainedUpdates = math.max(retainedUpdates, bounds[4])
+ retainedBytes = math.max(retainedBytes, bounds[5])
+ else
+ estimatesKnown = false
+ end
+ end
+ if key ~= current then redis.call('HDEL', KEYS[key], identity) end
+ end
+ end
+ if not estimatesKnown then
+ eventUpdateCount = nil
+ serializedEventBytes = nil
+ elseif not state then
+ eventUpdateCount = eventUpdateCount + retainedUpdates
+ serializedEventBytes = serializedEventBytes + retainedBytes
+ end
+ end
+ if state then
minStart = math.min(minStart, state.minStart)
maxStart = math.max(maxStart, state.maxStart)
- if state.eventUpdateCount and state.serializedEventBytes then
+ if eventUpdateCount and serializedEventBytes and state.eventUpdateCount and state.serializedEventBytes then
eventUpdateCount = eventUpdateCount + state.eventUpdateCount
serializedEventBytes = serializedEventBytes + state.serializedEventBytes
else
@@ -600,19 +649,24 @@ const TRACK_SCRIPT = `
end
end
redis.call('HSET', KEYS[2], member, cjson.encode({
- minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3],
+ minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3], historyBucket = bucket,
eventUpdateCount = eventUpdateCount, serializedEventBytes = serializedEventBytes
}))
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[1]), member)
+ redis.call('HSET', KEYS[current], identity,
+ cmsgpack.pack({minStart, maxStart, now + firstTtl, eventUpdateCount, serializedEventBytes}))
end
+ local firstExpiry = math.max((bucket + 1) * width + firstTtl,
+ redis.call('PEXPIRETIME', KEYS[current]))
+ -- Integer command arguments need decimal strings, including round timestamps.
+ redis.call('PEXPIREAT', KEYS[current], string.format('%.0f', firstExpiry))
local retention = tonumber(ARGV[2])
- local expired = expirePending(now, retention, ${CHUNK_SIZE})
- -- Bound abandoned state from the last admitted activity, independent of idle.
- -- A shorter policy on another writer must not shorten an existing expiry.
- local expiresAt = math.max(now + retention,
+ local expired = expirePending(now, retention, ${CLEANUP_SIZE})
+ -- The shared inactivity backstop covers the latest required member deadline.
+ local expiresAt = math.max(now + tonumber(ARGV[1]) + retention,
redis.call('PEXPIRETIME', KEYS[1]), redis.call('PEXPIRETIME', KEYS[2]))
- redis.call('PEXPIREAT', KEYS[1], expiresAt)
- redis.call('PEXPIREAT', KEYS[2], expiresAt)
+ redis.call('PEXPIREAT', KEYS[1], string.format('%.0f', expiresAt))
+ redis.call('PEXPIREAT', KEYS[2], string.format('%.0f', expiresAt))
return expired
`;
@@ -624,7 +678,7 @@ const SNAPSHOT_SCRIPT = `
local now = clock[1] * 1000 + math.floor(clock[2] / 1000)
local retention = tonumber(ARGV[1])
local cutoff = tonumber(ARGV[2]) or now
- local expired = expirePending(now, retention, ${CHUNK_SIZE})
+ local expired = expirePending(now, retention, ${CLEANUP_SIZE})
local pending = redis.call('ZCARD', KEYS[1])
local ready = math.max(0, redis.call('ZCOUNT', KEYS[1], '-inf', cutoff)
- redis.call('ZCOUNT', KEYS[1], '-inf', now - retention))
@@ -658,10 +712,32 @@ const HYDRATE_SCRIPT = `
// Returns one flag per member, in argument order: 1 when removed, 0 when a
// newer revision keeps the trace pending.
const ACKNOWLEDGE_SCRIPT = `
+ local clock = redis.call('TIME')
+ local now = clock[1] * 1000 + math.floor(clock[2] / 1000)
+ local width = tonumber(ARGV[2])
+ local bucket = math.floor(now / width)
+ local current = bucket - tonumber(ARGV[3]) + 3
+ if current < 3 or current > #KEYS then
+ return redis.error_reply('Trace readiness bucket clock changed; retry acknowledgement')
+ end
local removed = {}
- for i = 1, #ARGV, 2 do
+ local retained = false
+ for i = 4, #ARGV, 3 do
local state = redis.call('HGET', KEYS[2], ARGV[i])
- if state and cjson.decode(state).revision == ARGV[i + 1] then
+ local decoded = state and cjson.decode(state)
+ if decoded and decoded.revision == ARGV[i + 1] then
+ local identity = ARGV[i + 2]
+ if decoded.historyBucket then
+ local old = decoded.historyBucket - tonumber(ARGV[3]) + 3
+ if old >= 3 and old <= #KEYS and old ~= current then redis.call('HDEL', KEYS[old], identity) end
+ else
+ for key = 3, #KEYS do
+ if key ~= current then redis.call('HDEL', KEYS[key], identity) end
+ end
+ end
+ redis.call('HSET', KEYS[current], identity,
+ cmsgpack.pack({decoded.minStart, decoded.maxStart, now + tonumber(ARGV[1]), decoded.eventUpdateCount, decoded.serializedEventBytes}))
+ retained = true
redis.call('ZREM', KEYS[1], ARGV[i])
redis.call('HDEL', KEYS[2], ARGV[i])
table.insert(removed, 1)
@@ -669,9 +745,42 @@ const ACKNOWLEDGE_SCRIPT = `
table.insert(removed, 0)
end
end
+ if retained then
+ local expiresAt = math.max((bucket + 1) * width + tonumber(ARGV[1]),
+ redis.call('PEXPIRETIME', KEYS[current]))
+ redis.call('PEXPIREAT', KEYS[current], string.format('%.0f', expiresAt))
+ end
return removed
`;
+const readinessScriptHashes = new Map(
+ [TRACK_SCRIPT, SNAPSHOT_SCRIPT, HYDRATE_SCRIPT, ACKNOWLEDGE_SCRIPT].map(
+ (script) =>
+ [script, createHash("sha1").update(script).digest("hex")] as const,
+ ),
+);
+
+async function evalReadiness(
+ script: string,
+ keys: string[],
+ args: (string | number | Buffer)[],
+): Promise<unknown> {
+ if (!redis) throw new Error("Trace batching requires Redis");
+ try {
+ return await redis.evalsha(
+ readinessScriptHashes.get(script)!,
+ keys.length,
+ ...keys,
+ ...args,
+ );
+ } catch (error) {
+ // SCRIPT FLUSH or a promoted primary can have an empty script cache.
+ if (!(error instanceof Error) || !error.message.startsWith("NOSCRIPT"))
+ throw error;
+ return redis.eval(script, keys.length, ...keys, ...args);
+ }
+}
+
function recordTrackingVolume(
stage: "eligible" | "recorded",
entries: Iterable<
@@ -700,6 +809,7 @@ export async function trackTraceBatchActivity(
startTimeISO: string;
serializedEventBytes: number;
}[],
+ idleMs?: number,
): Promise<void> {
if (
!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
@@ -758,36 +868,106 @@ export async function trackTraceBatchActivity(
{ decision: "excluded" },
);
if (entries.length === 0) return;
+ const effectiveIdleMs = idleMs ?? (await resolveTraceIdleMs(projectId));
+ if (!Number.isSafeInteger(effectiveIdleMs) || effectiveIdleMs <= 0)
+ throw new Error("Trace batch idle time must be a positive integer");
if (!redis) throw new Error("Trace batching requires Redis");
- const idleMs = await resolveTraceIdleMs(projectId);
- // Bounded scripts keep ingestion from monopolizing the global Redis slot.
- for (let offset = 0; offset < entries.length; offset += CHUNK_SIZE) {
- const chunk = entries.slice(offset, offset + CHUNK_SIZE);
- const expired = Number(
- await redis.eval(
- TRACK_SCRIPT,
- 2,
- DUE_KEY,
- STATE_KEY,
- idleMs,
- env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
- ...chunk.flatMap(([traceId, state]) => [
- JSON.stringify([projectId, traceId]),
- state.minStart,
- state.maxStart,
- // Unique even after dispatch deletes and a later arrival recreates state.
- randomUUID(),
- state.eventUpdateCount,
- state.serializedEventBytes,
- ]),
- ),
- );
- // Count only acknowledged chunks; a timeout can leave Redis's outcome unknown.
- recordTrackingVolume("recorded", chunk);
- recordIncrement("langfuse.trace_batch.expired_traces", expired, {
- source: "ingestion",
- });
- recordIncrement("langfuse.trace_batch.tracked_traces", chunk.length);
+ const shards = new Map<number, typeof entries>();
+ for (const entry of entries) {
+ const shard = getTraceBatchShard(JSON.stringify([projectId, entry[0]]));
+ const group = shards.get(shard) ?? [];
+ group.push(entry);
+ shards.set(shard, group);
+ }
+ const clock = await redis.time();
+ const now = Number(clock[0]) * 1_000 + Math.floor(Number(clock[1]) / 1_000);
+ // Each pipeline contains bounded scripts from one cluster slot.
+ for (const [shard, group] of shards) {
+ const { due, state: stateKey } = getTraceBatchReadinessKeys(shard);
+ const first = getTraceBatchFirstKeys(shard, now);
+ const keys = [due, stateKey, ...first.keys];
+ for (
+ let offset = 0;
+ offset < group.length;
+ offset += REDIS_CHUNK_SIZE * REDIS_PIPELINE_SIZE
+ ) {
+ const pipeline =
+ group.length - offset > REDIS_CHUNK_SIZE
+ ? redis.pipeline()
+ : undefined;
+ const commands: {
+ chunk: typeof entries;
+ args: (string | number | Buffer)[];
+ }[] = [];
+ for (
+ let start = offset;
+ start <
+ Math.min(
+ group.length,
+ offset + REDIS_CHUNK_SIZE * REDIS_PIPELINE_SIZE,
+ );
+ start += REDIS_CHUNK_SIZE
+ ) {
+ const chunk = group.slice(start, start + REDIS_CHUNK_SIZE);
+ const args = [
+ effectiveIdleMs,
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS,
+ first.width,
+ first.firstBucket,
+ ...chunk.flatMap(([traceId, state]) => [
+ JSON.stringify([projectId, traceId]),
+ state.minStart,
+ state.maxStart,
+ // Unique even after dispatch deletes and a later arrival recreates state.
+ randomUUID(),
+ state.eventUpdateCount,
+ state.serializedEventBytes,
+ getTraceBatchFirstMember(JSON.stringify([projectId, traceId])),
+ ]),
+ ];
+ pipeline?.evalsha(
+ readinessScriptHashes.get(TRACK_SCRIPT)!,
+ keys.length,
+ ...keys,
+ ...args,
+ );
+ commands.push({ chunk, args });
+ }
+ const results: [Error | null, unknown][] | null = pipeline
+ ? await pipeline.exec()
+ : [[null, await evalReadiness(TRACK_SCRIPT, keys, commands[0].args)]];
+ if (!results)
+ throw new Error("Trace batch tracking pipeline returned no results");
+ let failure: unknown;
+ // A command error does not undo successful siblings. Record every known result.
+ for (const [index, { chunk, args }] of commands.entries()) {
+ try {
+ const [error, result] = results[index];
+ let expired: number;
+ if (error) {
+ if (!error.message.startsWith("NOSCRIPT")) throw error;
+ // Only this cache miss is safe to retry; successful siblings are not replayed.
+ expired = Number(
+ await redis.eval(TRACK_SCRIPT, keys.length, ...keys, ...args),
+ );
+ } else {
+ expired = Number(result);
+ }
+ recordTrackingVolume("recorded", chunk);
+ recordIncrement("langfuse.trace_batch.expired_traces", expired, {
+ source: "ingestion",
+ });
+ recordIncrement(
+ "langfuse.trace_batch.tracked_traces",
+ chunk.length,
+ );
+ } catch (error) {
+ if (failure === undefined) failure = error;
+ }
+ }
+ if (failure !== undefined) throw failure;
+ }
}
} catch (error) {
// This optional load experiment must not fail normal event ingestion.
@@ -884,59 +1064,91 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
if (!redis) throw new Error("Trace batching requires Redis");
const queue = TraceBatchQueue.getInstance();
if (!queue) throw new Error("Trace batch queue is unavailable");
- let cutoff = "";
- let now = 0;
- while (!this.stopping) {
- await this.extendLockOnProgress();
- const snapshot = (await redis.eval(
- SNAPSHOT_SCRIPT,
- 2,
- DUE_KEY,
- STATE_KEY,
- env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
+ const totals = { pending: 0, ready: 0, oldestAge: 0 };
+ const members: {
+ member: string;
+ projectId: string;
+ shard: number;
+ cutoff: string;
+ due: number;
+ }[] = [];
+ for (let shard = 0; shard < env.LANGFUSE_TRACE_BATCH_SHARD_COUNT; shard++) {
+ if (this.stopping) return;
+ const { due, state } = getTraceBatchReadinessKeys(shard);
+ let cutoff = "";
+ let now = 0;
+ while (!this.stopping) {
+ await this.extendLockOnProgress();
+ const snapshot = (await evalReadiness(
+ SNAPSHOT_SCRIPT,
+ [due, state],
+ [env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS, cutoff],
+ )) as (string | number)[];
+ const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);
+ now = clock;
+ cutoff = String(snapshot[1]);
+ recordIncrement("langfuse.trace_batch.expired_traces", expired, {
+ source: "dispatcher",
+ });
+ recordGauge("langfuse.trace_batch.pending_traces", pending, {
+ shard: String(shard),
+ });
+ recordGauge("langfuse.trace_batch.ready_traces", ready, {
+ shard: String(shard),
+ });
+ recordGauge(
+ "langfuse.trace_batch.oldest_due_age_ms",
+ Math.max(0, now - oldest),
+ { shard: String(shard) },
+ );
+ if (expired < CLEANUP_SIZE) {
+ totals.pending += pending;
+ totals.ready += ready;
+ totals.oldestAge = Math.max(
+ totals.oldestAge,
+ Math.max(0, now - oldest),
+ );
+ break;
+ }
+ }
+ if (this.stopping) return;
+ const dueMembers = await redis.zrange(
+ due,
+ `(${now - env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS}`,
cutoff,
- )) as (string | number)[];
- const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);
- now = clock;
- cutoff = String(snapshot[1]);
- recordIncrement("langfuse.trace_batch.expired_traces", expired, {
- source: "dispatcher",
- });
- recordGauge("langfuse.trace_batch.pending_traces", pending);
- recordGauge("langfuse.trace_batch.ready_traces", ready);
- recordGauge(
- "langfuse.trace_batch.oldest_due_age_ms",
- Math.max(0, now - oldest),
+ "BYSCORE",
+ "WITHSCORES",
);
- if (expired < CHUNK_SIZE) break;
+ recordDistribution(
+ "langfuse.trace_batch.snapshot_size",
+ dueMembers.length / 2,
+ );
+ for (let index = 0; index < dueMembers.length; index += 2) {
+ const member = dueMembers[index];
+ members.push({
+ member,
+ projectId: (JSON.parse(member) as [string, string])[0],
+ shard,
+ cutoff,
+ due: Number(dueMembers[index + 1]),
+ });
+ }
}
+ recordGauge("langfuse.trace_batch.pending_traces", totals.pending);
+ recordGauge("langfuse.trace_batch.ready_traces", totals.ready);
+ recordGauge("langfuse.trace_batch.oldest_due_age_ms", totals.oldestAge);
if (this.stopping) return;
- // One range read captures all due IDs without cursor races or loading every
- // trace's state. Memory and the Redis response size grow with the due cohort.
+ // Each range read captures one shard's due IDs. Pack the combined cohort so
+ // Redis placement does not determine the ClickHouse query boundaries.
const strategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;
- const members = (
- await redis.zrange(
- DUE_KEY,
- `(${now - env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS}`,
- cutoff,
- "BYSCORE",
- )
- ).map((member) => ({
- member,
- projectId: (JSON.parse(member) as [string, string])[0],
- }));
- // Stable sorting keeps Redis readiness order within each project.
- members.sort((a, b) => {
- if (a.projectId < b.projectId) {
- return -1;
- }
- if (a.projectId > b.projectId) {
- return 1;
- }
- return 0;
- });
- recordDistribution("langfuse.trace_batch.snapshot_size", members.length);
+ // Project packing needs due order within each project, including across shards.
+ members.sort(
+ (a, b) =>
+ compareStrings(a.projectId, b.projectId) ||
+ compareNumbers(a.due, b.due) ||
+ compareStrings(a.member, b.member),
+ );
const batchWeightLimits: TraceBatchWeightLimits = {
maxEventUpdates: toLimit(
env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES,
@@ -956,16 +1168,62 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
const acknowledge = async (
batch: readonly PendingTrace[],
- ): Promise<boolean[]> =>
- (
- (await redis!.eval(
- ACKNOWLEDGE_SCRIPT,
- 2,
- DUE_KEY,
- STATE_KEY,
- ...batch.flatMap(({ member, trace }) => [member, trace.revision]),
- )) as number[]
- ).map((flag) => Number(flag) === 1);
+ ): Promise<boolean[]> => {
+ const removed = Array<boolean>(batch.length).fill(false);
+ const groups = new Map<
+ number,
+ { entry: PendingTrace; index: number }[]
+ >();
+ batch.forEach((entry, index) => {
+ const shard = getTraceBatchShard(entry.member);
+ const group = groups.get(shard) ?? [];
+ group.push({ entry, index });
+ groups.set(shard, group);
+ });
+ for (const [shard, group] of groups) {
+ const { due, state } = getTraceBatchReadinessKeys(shard);
+ for (
+ let offset = 0;
+ offset < group.length;
+ offset += REDIS_CHUNK_SIZE
+ ) {
+ const chunk = group.slice(offset, offset + REDIS_CHUNK_SIZE);
+ const clock = await redis!.time();
+ const first = getTraceBatchFirstKeys(
+ shard,
+ Number(clock[0]) * 1_000 + Math.floor(Number(clock[1]) / 1_000),
+ );
+ const flags = (await evalReadiness(
+ ACKNOWLEDGE_SCRIPT,
+ [due, state, ...first.keys],
+ [
+ env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS,
+ first.width,
+ first.firstBucket,
+ ...chunk.flatMap(({ entry: { member, trace } }) => [
+ member,
+ trace.revision,
+ getTraceBatchFirstMember(member),
+ ]),
+ ],
+ )) as number[];
+ chunk.forEach(({ index }, position) => {
+ removed[index] = Number(flags[position]) === 1;
+ });
+ }
+ }
+ return removed;
+ };
+
+ const enqueued: PendingTrace[] = [];
+ const flushEnqueued = async () => {
+ if (enqueued.length === 0) return;
+ const removed = await acknowledge(enqueued.splice(0));
+ recordIncrement(
+ "langfuse.trace_batch.reactivated_traces",
+ removed.filter((wasRemoved) => !wasRemoved).length,
+ );
+ };
// Excluded traces leave Redis like dispatched ones; activity after this
// run's snapshot starts a fresh pending window that is evaluated again.
@@ -1017,6 +1275,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
},
{ jobId: id },
);
+ enqueued.push(...batch);
recordDistribution("langfuse.trace_batch.size", batch.length);
recordDistribution(
"langfuse.trace_batch.project_count",
@@ -1088,11 +1347,6 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
strategy,
});
}
- const removed = await acknowledge(batch);
- recordIncrement(
- "langfuse.trace_batch.reactivated_traces",
- removed.filter((wasRemoved) => !wasRemoved).length,
- );
};
let projectBatchTail: PendingTrace[] = [];
@@ -1104,32 +1358,54 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
) {
await this.extendLockOnProgress();
const candidates = members.slice(offset, offset + CHUNK_SIZE);
- const hydrated = (await redis.eval(
- HYDRATE_SCRIPT,
- 2,
- DUE_KEY,
- STATE_KEY,
- cutoff,
- env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
- ...candidates.map(({ member }) => member),
- )) as string[];
+ const byShard = new Map<number, typeof candidates>();
+ for (const candidate of candidates) {
+ const group = byShard.get(candidate.shard) ?? [];
+ group.push(candidate);
+ byShard.set(candidate.shard, group);
+ }
+ const hydratedByMember = new Map<string, [string, string]>();
+ for (const [shard, group] of byShard) {
+ const { due, state } = getTraceBatchReadinessKeys(shard);
+ for (let start = 0; start < group.length; start += REDIS_CHUNK_SIZE) {
+ await this.extendLockOnProgress();
+ const part = (await evalReadiness(
+ HYDRATE_SCRIPT,
+ [due, state],
+ [
+ group[0].cutoff,
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
+ ...group
+ .slice(start, start + REDIS_CHUNK_SIZE)
+ .map(({ member }) => member),
+ ],
+ )) as string[];
+ for (let index = 0; index < part.length; index += 3) {
+ hydratedByMember.set(part[index], [
+ part[index + 1],
+ part[index + 2],
+ ]);
+ }
+ }
+ }
recordIncrement(
"langfuse.trace_batch.skipped_traces",
- candidates.length - hydrated.length / 3,
+ candidates.length - hydratedByMember.size,
);
const hydratedCandidates: PendingTrace[] = [];
const excluded: {
entry: PendingTrace;
reason: TraceExclusionReason;
}[] = [];
- for (let i = 0; i < hydrated.length; i += 3) {
+ for (const { member } of candidates) {
if (this.stopping) return;
- const member = hydrated[i];
+ const hydrated = hydratedByMember.get(member);
+ if (!hydrated) continue;
const [projectId, traceId] = JSON.parse(member) as [string, string];
- const state = JSON.parse(hydrated[i + 1]);
+ const state = JSON.parse(hydrated[0]);
const entry: PendingTrace = {
member,
- due: Number(hydrated[i + 2]),
+ due: Number(hydrated[1]),
estimates:
Number.isSafeInteger(state.eventUpdateCount) &&
state.eventUpdateCount >= 0 &&
@@ -1179,49 +1455,59 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
{ strategy },
);
- if (strategy === "project") {
- projectBatchTail = [];
- if (selected.at(-1)?.length !== env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {
- projectBatchTail = selected.pop() ?? [];
+ try {
+ if (strategy === "project") {
+ projectBatchTail = [];
+ if (selected.at(-1)?.length !== env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {
+ projectBatchTail = selected.pop() ?? [];
+ }
+ for (const batch of selected) {
+ if (this.stopping) return;
+ await enqueue(batch);
+ }
+ continue;
+ }
+
+ const fullBatches = selected.filter(
+ (batch) => batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
+ );
+ const partialBatches = selected.filter(
+ (batch) => batch.length < env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
+ );
+ for (const batch of fullBatches) {
+ if (this.stopping) return;
+ await enqueue(batch);
}
- for (const batch of selected) {
+ await this.extendLockOnProgress();
+ if (this.stopping) return;
+ const prepared = prepareLocalityPartials(
+ partialBatches,
+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
+ batchWeightLimits,
+ );
+ localityPartials = prepared.carry;
+ for (const batch of prepared.dispatch) {
if (this.stopping) return;
await enqueue(batch);
}
- continue;
+ } finally {
+ await flushEnqueued();
}
-
- const fullBatches = selected.filter(
- (batch) => batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
- );
- const partialBatches = selected.filter(
- (batch) => batch.length < env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
- );
- for (const batch of fullBatches) {
- if (this.stopping) return;
- await enqueue(batch);
+ }
+ if (this.stopping) return;
+ try {
+ if (strategy === "project") {
+ if (projectBatchTail.length > 0) await enqueue(projectBatchTail);
+ return;
}
- await this.extendLockOnProgress();
- if (this.stopping) return;
- const prepared = prepareLocalityPartials(
- partialBatches,
- env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
- batchWeightLimits,
- );
- localityPartials = prepared.carry;
- for (const batch of prepared.dispatch) {
+ for (const batch of localityPartials.toSorted(
+ compareBatchesByOldestDue,
+ )) {
if (this.stopping) return;
await enqueue(batch);
}
- }
- if (this.stopping) return;
- if (strategy === "project") {
- if (projectBatchTail.length > 0) await enqueue(projectBatchTail);
- return;
- }
- for (const batch of localityPartials.toSorted(compareBatchesByOldestDue)) {
- if (this.stopping) return;
- await enqueue(batch);
+ } finally {
+ await flushEnqueued();
}
}
}
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index 6c2423ad95..8adea51fdd 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -97,6 +97,7 @@ async function loadProjectRule(projectId: string) {
async function summarizeTraceBatch(
observations: Observation[],
topics: TopicsBatchProjects,
+ jobEnqueuedAt: Date,
): Promise<TraceOutcome> {
const first = observations[0];
const traceId = first?.traceId;
@@ -130,6 +131,7 @@ async function summarizeTraceBatch(
projectId: first.projectId,
traceId,
traceTimestamp,
+ jobEnqueuedAt,
environment: first.environment,
traceName: first.name ?? "",
transcript,
@@ -316,7 +318,7 @@ export const traceBatchQueueProcessor: Processor<
);
if (!summaryFailed)
topicsOutcomes.push(
- await summarizeTraceBatch(observations, topics),
+ await summarizeTraceBatch(observations, topics, event.timestamp),
);
})
.catch((error: unknown) => {