Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): add opt-in trace batch locality selection (#17498)

Lysander Kiesele1ac230adb25
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 992936e6de..607bf73720 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1082,6 +1082,9 @@ importers:
'@mastra/mcp':
specifier: 1.8.1
+ '@node-rs/xxhash':
+ specifier: 1.7.7
+ version: 1.7.7
'@opentelemetry/api':
specifier: ^1.9.1
version: 1.9.1
@@ -3459,6 +3462,92 @@ packages:
'@nodable/[email protected]':
resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==}
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-9uOAXzMmayilkSmoSgrtZDKG2a4E8I0f58Ydm0/us2ubfUGqBr6AjaxzRjipuDBGeNZYYQP1wmoUzLAmuUajIQ==}
+ engines: {node: '>= 12'}
+ cpu: [arm]
+ os: [android]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-NDqhut7aniQegfLEYBDxmhI5gFHcDur3D9G7E724SzF3aPw0lTAN9cLixd/Cs+BJRwq3oziOdGtYW9j0xM/B0Q==}
+ engines: {node: '>= 12'}
+ cpu: [arm64]
+ os: [android]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-uwrEQELN50J2leUNBTir50nRxJJT3Dts3WDnvEij+MMJco++uI2dXoHbwP/UoavGuyoPtngrWxKP8XS9/mLvOA==}
+ engines: {node: '>= 12'}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-y7AUeH573Ks5BDaJtmyvPxHhPDGedwZnU/hqFbS/DLUqpYGxXLwYkV5iSBtEfFnYyyae1MVjUBqSaCXB6wuLDg==}
+ engines: {node: '>= 12'}
+ cpu: [x64]
+ os: [darwin]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-4kOYAbnPcFgcpOVQvthha09hjlmswIsfcZWm8uTZC1sau3tdynyyQy1/OcOG03EdP1W0qHOC3ORfUTS9oxbI9A==}
+ engines: {node: '>= 12'}
+ cpu: [x64]
+ os: [freebsd]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-zO9DsqETMS+8/UarCugj1NW+PcD4haFfYQuVXFaPTskgtocvFY8nYY0N59YZukBWlSGh3xBtXPMVd39zeRiNKg==}
+ engines: {node: '>= 12'}
+ cpu: [arm]
+ os: [linux]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-48kFnEIy8w1R8pGKblG+em6Q7YuDRWcLsSRpFCIrbpsA6aAQdvZnMXiyx+HSq5gEWwM35Hrb30L1sStUSetFmw==}
+ engines: {node: '>= 12'}
+ cpu: [arm64]
+ os: [linux]
+ libc: [glibc]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-m1Wz3ySODY7XZixTgLn9n59tUVSTmhO4iwcTLX5lRNnJaCOiWywsySPPbo3++oebSGTwAdJEmPJq5BLPSEZGGQ==}
+ engines: {node: '>= 12'}
+ cpu: [arm64]
+ os: [linux]
+ libc: [musl]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-rdLsrNjOUbOCdsdZZqfqeE24y/yeSR8c11w5Xkwj/Qkrcpaq3qB5f3aIW+6WuQJjdBsJwOxHCspSuUiaY6Xx4g==}
+ engines: {node: '>= 12'}
+ cpu: [x64]
+ os: [linux]
+ libc: [glibc]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-1LmaKDAIoqZcSQJG22cZMH9aL4B6jg5kQygg6pY7NyICzw+taLPLHhaMMEE+E0n2BodRXzVjZvF15pxi4qXtug==}
+ engines: {node: '>= 12'}
+ cpu: [x64]
+ os: [linux]
+ libc: [musl]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-F2QaHmplRZbg1T/AlQuW11MmRsxLvNVEsofeQd92/u704L5+BBEVc13MutJwioXNIAW1GmgOpfwVG6ZGeLAoyA==}
+ engines: {node: '>= 12'}
+ cpu: [arm64]
+ os: [win32]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-YnHHBQVHju1Sx8vc2s2RauAGXAwWeqFMC4CvaJJ8j5WPnW2x8p9hTQRURMl/ATPLp/t8j4C9AEmLNtrNdE6JXQ==}
+ engines: {node: '>= 12'}
+ cpu: [ia32]
+ os: [win32]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-0uVSktuJUetF467YZwHoGuBGcY8RF30DixboXTgMYG5amWze2j+OvaSpQKLsHhf0fs2UZH9W4KQG3SfUizlmGA==}
+ engines: {node: '>= 12'}
+ cpu: [x64]
+ os: [win32]
+
+ '@node-rs/[email protected]':
+ resolution: {integrity: sha512-lQflJcUTqTvIbFNvq+6DnXQr2MCAqEBsXt9IgTiuhQeaFPejoUJBRF2WlfU5hfUzsCjU+DrzZlxzob0Q4zdHLw==}
+ engines: {node: '>= 12'}
+
'@nodelib/[email protected]':
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
engines: {node: '>= 8'}
@@ -15169,6 +15258,61 @@ snapshots:
'@nodable/[email protected]': {}
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optional: true
+
+ '@node-rs/[email protected]':
+ optionalDependencies:
+ '@node-rs/xxhash-android-arm-eabi': 1.7.7
+ '@node-rs/xxhash-android-arm64': 1.7.7
+ '@node-rs/xxhash-darwin-arm64': 1.7.7
+ '@node-rs/xxhash-darwin-x64': 1.7.7
+ '@node-rs/xxhash-freebsd-x64': 1.7.7
+ '@node-rs/xxhash-linux-arm-gnueabihf': 1.7.7
+ '@node-rs/xxhash-linux-arm64-gnu': 1.7.7
+ '@node-rs/xxhash-linux-arm64-musl': 1.7.7
+ '@node-rs/xxhash-linux-x64-gnu': 1.7.7
+ '@node-rs/xxhash-linux-x64-musl': 1.7.7
+ '@node-rs/xxhash-win32-arm64-msvc': 1.7.7
+ '@node-rs/xxhash-win32-ia32-msvc': 1.7.7
+ '@node-rs/xxhash-win32-x64-msvc': 1.7.7
+
'@nodelib/[email protected]':
dependencies:
'@nodelib/fs.stat': 2.0.5
diff --git a/worker/package.json b/worker/package.json
index 2997984744..a9c7e78d54 100644
--- a/worker/package.json
+++ b/worker/package.json
@@ -38,6 +38,7 @@
"@appsignal/opentelemetry-instrumentation-bullmq": "^0.8.0",
"@langfuse/native": "workspace:*",
"@langfuse/shared": "workspace:*",
+ "@node-rs/xxhash": "1.7.7",
"@aws-sdk/client-lambda-microvms": "catalog:",
"@mastra/core": "1.46.0",
"@mastra/mcp": "1.8.1",
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
index e52ccf7f4c..0e532336a7 100644
--- a/worker/src/__tests__/traceBatching.test.ts
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -26,6 +26,8 @@ import {
import * as shared from "@langfuse/shared/src/server";
import { env } from "../env";
import {
+ selectTraceBatches,
+ type PendingTrace,
trackTraceBatchActivity,
TraceBatchDispatcher,
} from "../features/traceBatching/traceBatching";
@@ -54,6 +56,221 @@ vi.mock("../features/evaluation/observationEval", async (importOriginal) => ({
fetchObservationEvalRules: vi.fn().mockResolvedValue([]),
}));
+describe("trace batch selection", () => {
+ const minute = 60_000;
+ const pendingTrace = (
+ projectId: string,
+ traceId: string,
+ due: number,
+ minStart: number,
+ maxStart: number,
+ ): PendingTrace => ({
+ member: JSON.stringify([projectId, traceId]),
+ due,
+ trace: {
+ projectId,
+ traceId,
+ minStart,
+ maxStart,
+ revision: `revision-${projectId}-${traceId}`,
+ },
+ });
+ const ids = (batches: PendingTrace[][]) =>
+ batches.map((batch) => batch.map(({ member }) => member));
+
+ it("orders batches by the events table locality key", () => {
+ const projectB = pendingTrace("project-b", "trace-1", 1, 0, minute);
+ const laterMinute = pendingTrace(
+ "project-a",
+ "trace-3",
+ 2,
+ 2 * minute,
+ 3 * minute,
+ );
+ const widerRange = pendingTrace(
+ "project-a",
+ "trace-4",
+ 3,
+ 10_000,
+ 2 * minute,
+ );
+ const trace0 = pendingTrace("project-a", "trace-0", 4, 20_000, 50_000);
+ const trace1 = pendingTrace("project-a", "trace-1", 5, 10_000, 50_000);
+
+ const batches = selectTraceBatches(
+ [projectB, laterMinute, widerRange, trace0, trace1],
+ 2,
+ "locality",
+ );
+
+ expect(ids(batches)).toEqual([
+ [trace1.member, trace0.member],
+ [widerRange.member, laterMinute.member],
+ [projectB.member],
+ ]);
+ });
+
+ it("is independent of readiness and input order", () => {
+ const candidates = [
+ pendingTrace("project-b", "trace-3", 1, 10 * minute, 11 * minute),
+ pendingTrace("project-a", "trace-0", 4, 10 * minute, 11 * minute),
+ pendingTrace("project-a", "trace-1", 3, 10 * minute, 11 * minute),
+ pendingTrace("project-a", "trace-4", 2, 10 * minute, 11 * minute),
+ ];
+ const expected = selectTraceBatches(candidates, 3, "locality");
+
+ expect(expected.flat().map(({ trace }) => trace.traceId)).toEqual([
+ "trace-1",
+ "trace-0",
+ "trace-4",
+ "trace-3",
+ ]);
+ expect(selectTraceBatches(candidates.toReversed(), 3, "locality")).toEqual(
+ expected,
+ );
+ });
+
+ it("moves a batch boundary instead of bridging distant time ranges", () => {
+ const early = Array.from({ length: 59 }, (_, index) =>
+ pendingTrace("project", `early-${index}`, index, 0, minute),
+ );
+ const late = Array.from({ length: 60 }, (_, index) =>
+ pendingTrace(
+ "project",
+ `late-${index}`,
+ 100 + index,
+ 10_000 * minute,
+ 10_001 * minute,
+ ),
+ );
+
+ const batches = selectTraceBatches([...late, ...early], 60, "locality");
+
+ expect(batches.map((batch) => batch.length)).toEqual([59, 60]);
+ expect(
+ batches[0].every(({ trace }) => trace.traceId.startsWith("early-")),
+ ).toBe(true);
+ expect(
+ batches[1].every(({ trace }) => trace.traceId.startsWith("late-")),
+ ).toBe(true);
+ });
+
+ it("avoids a cross-project fill when the same job count permits it", () => {
+ const projectA = Array.from({ length: 59 }, (_, index) =>
+ pendingTrace("project-a", `trace-${index}`, index, 0, minute),
+ );
+ const projectB = Array.from({ length: 60 }, (_, index) =>
+ pendingTrace("project-b", `trace-${index}`, 100 + index, 0, minute),
+ );
+
+ const batches = selectTraceBatches(
+ [...projectB, ...projectA],
+ 60,
+ "locality",
+ );
+
+ expect(batches.map((batch) => batch.length)).toEqual([59, 60]);
+ expect(
+ batches.map(
+ (batch) => new Set(batch.map(({ trace }) => trace.projectId)).size,
+ ),
+ ).toEqual([1, 1]);
+ });
+
+ it("cuts at the largest trace-hash gap within one time range", () => {
+ const candidates = Array.from({ length: 5 }, (_, index) =>
+ pendingTrace("project", `trace-${index}`, index, 0, minute),
+ );
+
+ const batches = selectTraceBatches(candidates.toReversed(), 3, "locality");
+
+ expect(
+ batches.map((batch) => batch.map(({ trace }) => trace.traceId)),
+ ).toEqual([
+ ["trace-1", "trace-0"],
+ ["trace-4", "trace-2", "trace-3"],
+ ]);
+ });
+
+ it("splits distant narrow traces rather than sharing one envelope", () => {
+ const candidates = Array.from({ length: 3 }, (_, index) =>
+ pendingTrace(
+ "project",
+ `day-${index}`,
+ index,
+ index * 3 * 60 * minute,
+ index * 3 * 60 * minute + minute,
+ ),
+ );
+
+ expect(ids(selectTraceBatches(candidates, 60, "locality"))).toEqual(
+ candidates.map((candidate) => [candidate.member]),
+ );
+ });
+
+ it("preserves uneven cohorts separated by mandatory envelope boundaries", () => {
+ const cohorts = [2, 1, 4, 2].map((size, cohort) =>
+ Array.from({ length: size }, (_, index) => {
+ const start = (cohort * 120 + index) * minute;
+ return pendingTrace(
+ "project",
+ `cohort-${cohort}-${index}`,
+ index,
+ start,
+ start + minute,
+ );
+ }),
+ );
+
+ for (const cap of [4, 9]) {
+ expect(
+ ids(selectTraceBatches(cohorts.flat().toReversed(), cap, "locality")),
+ ).toEqual(ids(cohorts));
+ }
+ });
+
+ it("bounds chained wide-trace expansion relative to the batch seed", () => {
+ const seed = pendingTrace("project", "seed", 1, 0, 120 * minute);
+ const compatible = pendingTrace(
+ "project",
+ "compatible",
+ 2,
+ 0,
+ 144 * minute,
+ );
+ const wouldChain = pendingTrace(
+ "project",
+ "would-chain",
+ 3,
+ 0,
+ 168 * minute,
+ );
+
+ const batches = selectTraceBatches(
+ [wouldChain, compatible, seed],
+ 60,
+ "locality",
+ );
+
+ expect(batches.flat()).toHaveLength(3);
+ expect(
+ batches.some((batch) => {
+ const ids = new Set(batch.map(({ member }) => member));
+ return ids.has(seed.member) && ids.has(wouldChain.member);
+ }),
+ ).toBe(false);
+ expect(
+ batches.every((batch) => {
+ const union =
+ Math.max(...batch.map(({ trace }) => trace.maxStart)) -
+ Math.min(...batch.map(({ trace }) => trace.minStart));
+ const seedSpan = batch[0].trace.maxStart - batch[0].trace.minStart;
+ return union <= Math.max(60 * minute, seedSpan * 1.25);
+ }),
+ ).toBe(true);
+ });
+});
+
describe("trace micro-batch scheduling with Redis", () => {
const dueKey = "{trace-batch}:due";
const stateKey = "{trace-batch}:state";
@@ -62,6 +279,7 @@ describe("trace micro-batch scheduling with Redis", () => {
const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
const originalSamplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;
+ const originalStrategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;
const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;
const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;
const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;
@@ -98,6 +316,7 @@ describe("trace micro-batch scheduling with Redis", () => {
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;
+ env.LANGFUSE_TRACE_BATCH_STRATEGY = "project";
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 60;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 7_200_000;
await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");
@@ -120,6 +339,7 @@ describe("trace micro-batch scheduling with Redis", () => {
env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = originalSamplingRate;
+ env.LANGFUSE_TRACE_BATCH_STRATEGY = originalStrategy;
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;
env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;
@@ -130,6 +350,70 @@ describe("trace micro-batch scheduling with Redis", () => {
});
afterAll(() => client().disconnect());
+ it("dispatches with locality selection and records bounded selector measurements", async () => {
+ env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";
+ const minute = 60_000;
+ await trackTraceBatchActivity("project-a", [
+ event("short-1", 10 * minute),
+ event("short-1", 12 * minute),
+ event("wide", 0),
+ event("wide", 8 * 60 * minute),
+ ]);
+ await trackTraceBatchActivity("project-b", [
+ event("short-2", 11 * minute),
+ event("short-2", 13 * minute),
+ ]);
+ const due = Date.now() - 1_000;
+ await client().zadd(
+ dueKey,
+ due,
+ member("project-a", "short-1"),
+ due + 1,
+ member("project-a", "wide"),
+ due + 2,
+ member("project-b", "short-2"),
+ );
+
+ await runner().processBatch();
+
+ const jobs = await queue.getJobs(["wait"]);
+ expect(
+ jobs
+ .map((job) =>
+ job.data.payload.traces
+ .map(({ traceId }) => traceId)
+ .sort()
+ .join(","),
+ )
+ .sort(),
+ ).toEqual(["short-1,short-2,wide"]);
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.candidate_buffer_size",
+ 3,
+ { strategy: "locality" },
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.event_time_envelope_ms",
+ 29_040_000,
+ { strategy: "locality" },
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.dispatched_batches",
+ 1,
+ { strategy: "locality", fill: "partial" },
+ );
+ expect(
+ vi
+ .mocked(recordDistribution)
+ .mock.calls.some(
+ ([name, value, tags]) =>
+ name === "langfuse.trace_batch.selector_duration_ms" &&
+ Number(value) >= 0 &&
+ tags?.strategy === "locality",
+ ),
+ ).toBe(true);
+ });
+
it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
const evaluate = vi.spyOn(client(), "eval");
@@ -500,47 +784,79 @@ describe("trace micro-batch scheduling with Redis", () => {
},
);
- it("groups the complete due cohort by sorted project across hydration chunks and the former run limit", async () => {
- const due = Date.now() - 100_000;
- const traces = Array.from({ length: 10_061 }, (_, i) => ({
- projectId: ["project-c", "project-a", "project-b"][i % 3],
- traceId: `trace-${String(i).padStart(5, "0")}`,
- due: due + Math.floor(i / 2_000) * 1_000,
- }));
- for (const projectId of ["project-c", "project-a", "project-b"]) {
- await trackTraceBatchActivity(
- projectId,
- traces
- .filter((trace) => trace.projectId === projectId)
- .map((trace) => event(trace.traceId)),
+ it.each(["project", "locality"] as const)(
+ "dispatches the complete due cohort across hydration chunks with %s",
+ async (strategy) => {
+ env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;
+ const due = Date.now() - 100_000;
+ const traces = Array.from({ length: 10_061 }, (_, i) => ({
+ projectId: ["project-c", "project-a", "project-b"][i % 3],
+ traceId: `trace-${String(i).padStart(5, "0")}`,
+ due: due + Math.floor(i / 2_000) * 1_000,
+ }));
+ for (const projectId of ["project-c", "project-a", "project-b"]) {
+ await trackTraceBatchActivity(
+ projectId,
+ traces
+ .filter((trace) => trace.projectId === projectId)
+ .map((trace) => event(trace.traceId)),
+ );
+ }
+ await client().zadd(
+ dueKey,
+ ...traces.flatMap((trace) => [
+ trace.due,
+ member(trace.projectId, trace.traceId),
+ ]),
);
- }
- await client().zadd(
- dueKey,
- ...traces.flatMap((trace) => [
- trace.due,
- member(trace.projectId, trace.traceId),
- ]),
- );
- const add = vi.spyOn(queue, "add");
- await runner().processBatch();
- const batches = add.mock.calls.map(([, job]) => job.payload.traces);
- expect(batches.map((batch) => batch.length)).toEqual([
- ...Array(167).fill(60),
- 41,
- ]);
- const expected = traces.sort(
- (a, b) =>
- a.projectId.localeCompare(b.projectId) ||
- a.due - b.due ||
- a.traceId.localeCompare(b.traceId),
- );
- expect(
- batches.flat().map((trace) => member(trace.projectId, trace.traceId)),
- ).toEqual(expected.map((trace) => member(trace.projectId, trace.traceId)));
- expect(await client().zcard(dueKey)).toBe(0);
- expect(await client().hlen(stateKey)).toBe(0);
- });
+ const add = vi.spyOn(queue, "add");
+ await runner().processBatch();
+ const batches = add.mock.calls.map(([, job]) => job.payload.traces);
+ const assigned = batches
+ .flat()
+ .map((trace) => member(trace.projectId, trace.traceId));
+ expect(assigned).toHaveLength(traces.length);
+ expect(new Set(assigned)).toEqual(
+ new Set(traces.map((trace) => member(trace.projectId, trace.traceId))),
+ );
+ expect(batches.every((batch) => batch.length <= 60)).toBe(true);
+ const candidateSizes = vi
+ .mocked(recordDistribution)
+ .mock.calls.filter(
+ ([name]) => name === "langfuse.trace_batch.candidate_buffer_size",
+ )
+ .map(([, value]) => Number(value));
+ expect(Math.max(...candidateSizes)).toBeLessThanOrEqual(
+ strategy === "project" ? 1059 : 1000,
+ );
+ if (strategy === "project") {
+ expect(batches.map((batch) => batch.length)).toEqual([
+ ...Array(167).fill(60),
+ 41,
+ ]);
+ const expected = traces.sort(
+ (a, b) =>
+ a.projectId.localeCompare(b.projectId) ||
+ a.due - b.due ||
+ a.traceId.localeCompare(b.traceId),
+ );
+ expect(
+ batches.flat().map((trace) => member(trace.projectId, trace.traceId)),
+ ).toEqual(
+ expected.map((trace) => member(trace.projectId, trace.traceId)),
+ );
+ } else {
+ expect(batches.length).toBeGreaterThanOrEqual(
+ Math.ceil(traces.length / 60),
+ );
+ expect(batches.length).toBeLessThanOrEqual(
+ Math.ceil(traces.length / 1000) * Math.ceil(1000 / 60),
+ );
+ }
+ expect(await client().zcard(dueKey)).toBe(0);
+ expect(await client().hlen(stateKey)).toBe(0);
+ },
+ );
it("revalidates the due ID list after deletion, missing state, reactivation and new arrivals", async () => {
const traces = Array.from(
@@ -778,20 +1094,24 @@ describe("trace micro-batch scheduling with Redis", () => {
).toBe(10_001);
});
- it("keeps due state on enqueue failure and retries it on the next dispatch", async () => {
- await trackTraceBatchActivity("project", [event("trace")]);
- await makeDue("project", "trace");
- vi.spyOn(queue, "add").mockRejectedValueOnce(
- new Error("enqueue unavailable"),
- );
- const dispatcher = runner();
- await dispatcher.processBatch();
- expect(await client().zcard(dueKey)).toBe(1);
- expect(await client().hlen(stateKey)).toBe(1);
- await dispatcher.processBatch();
- expect(await queue.getWaitingCount()).toBe(1);
- expect(await client().zcard(dueKey)).toBe(0);
- });
+ it.each(["project", "locality"] as const)(
+ "keeps due state on enqueue failure and retries it with %s",
+ async (strategy) => {
+ env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;
+ await trackTraceBatchActivity("project", [event("trace")]);
+ await makeDue("project", "trace");
+ vi.spyOn(queue, "add").mockRejectedValueOnce(
+ new Error("enqueue unavailable"),
+ );
+ const dispatcher = runner();
+ await dispatcher.processBatch();
+ expect(await client().zcard(dueKey)).toBe(1);
+ expect(await client().hlen(stateKey)).toBe(1);
+ await dispatcher.processBatch();
+ expect(await queue.getWaitingCount()).toBe(1);
+ expect(await client().zcard(dueKey)).toBe(0);
+ },
+ );
it.each(["updated", "deleted", "expired"] as const)(
"preserves arrivals during enqueue, including deleted/recreated state: %s",
diff --git a/worker/src/env.ts b/worker/src/env.ts
index ac19d4a74c..5bd76cfa9a 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -132,6 +132,9 @@ const EnvSchema = z.object({
.positive()
.max(1_000)
.default(60),
+ LANGFUSE_TRACE_BATCH_STRATEGY: z
+ .enum(["project", "locality"])
+ .default("project"),
LANGFUSE_TRACE_BATCH_IDLE_MS: z.coerce
.number()
.int()
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index e8e3c23207..f1475e72eb 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -14,6 +14,7 @@ changes. These internal controls are intentionally absent from env templates.
| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |
+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 1,000. |
| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |
@@ -67,8 +68,33 @@ window prune reads. `TRACE_QUERY_BUFFER_MS` pads the earliest and latest recorde
start times by two minutes each. This fixed query margin can include observations
outside the recorded bounds; it neither waits for arrivals nor guarantees trace
completeness. Query limits are fixed at two threads and 30 seconds with
-timeouts failing the job. This baseline has no hash-locality grouping,
-per-trace windows, query tuning controls or allocation based on trace size.
+timeouts failing the job. Per-trace windows, query tuning controls and allocation
+based on trace size are separate experiments.
+
+### Opt-in locality grouping
+
+`LANGFUSE_TRACE_BATCH_STRATEGY=locality` changes grouping only after dispatch is
+explicitly enabled in a cloud region. The default `project` strategy preserves
+project ordering and carries unfinished jobs across hydration chunks.
+
+Locality sorts each hydrated window by project, minimum start minute, maximum
+start minute and `xxHash32(traceId)`. It finds the fewest feasible jobs under the
+trace cap and an event-time envelope of at most one hour, or 125% of the first
+trace's observed span if larger. Within that job count, it minimizes project
+boundaries, then minute × trace count, then gaps between trace hashes. These are
+read-locality proxies, not measured ClickHouse scan costs.
+
+Locality selections dispatch immediately per window of at most 1,000 candidates.
+Partials do not carry between windows, which can increase small jobs. The full
+ready-ID snapshot remains unchanged; only selector input is bounded. Selector
+cost is O(n × k × cap), where k is the fewest feasible jobs. Measure duration
+before increasing scale, especially with distant event times that force many jobs.
+
+Compare `dispatched_batches` (tags `strategy`, `fill`),
+`event_time_envelope_ms`, `observed_start_span_ms`, `candidate_buffer_size` and
+`selector_duration_ms` under `langfuse.trace_batch`. The envelope metric includes
+the reader's two-minute margin on each side; the selector's envelope limit does
+not. Payload and Redis state are unchanged.
## Monitor dispatcher memory
@@ -113,5 +139,5 @@ lazy retention rules. Drain and expired jobs are removed immediately on success.
For an already enabled cloud experiment, explicitly set the new read flag to
`true` when deploying this version if reads should continue; leaving it unset
drains instead. Upgrade all producers before relying on native expiry: older
-producers do not install or refresh that TTL. Locality, query controls and load
+producers do not install or refresh that TTL. Locality partial carry, query controls and load
test artifacts belong in separate follow-up changes.
diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.ts
index a2dfa70155..d8af2ca339 100644
--- a/worker/src/features/traceBatching/traceBatching.ts
+++ b/worker/src/features/traceBatching/traceBatching.ts
@@ -1,4 +1,5 @@
import { createHash, randomUUID } from "node:crypto";
+import { xxh32 } from "@node-rs/xxhash";
import {
logger,
QueueJobs,
@@ -23,12 +24,256 @@ import {
const DUE_KEY = "{trace-batch}:due";
const STATE_KEY = "{trace-batch}:state";
const CHUNK_SIZE = 1_000;
-type PendingTrace = {
+const QUERY_BUFFER_MS = 2 * 60_000;
+const MINUTE_MS = 60_000;
+const NARROW_BATCH_MAX_ENVELOPE_MS = 60 * MINUTE_MS;
+const WIDE_MAX_EXPANSION_RATIO = 0.25;
+
+export type TraceBatchStrategy = "project" | "locality";
+export type PendingTrace = {
member: string;
due: number;
trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];
};
+const compareNumbers = (left: number, right: number) => left - right;
+
+const compareStrings = (left: string, right: string) =>
+ left < right ? -1 : left > right ? 1 : 0;
+
+const toMinuteBucket = (timestamp: number) => Math.floor(timestamp / MINUTE_MS);
+
+const getSpanMs = ({ trace }: PendingTrace) => trace.maxStart - trace.minStart;
+
+const allowedEnvelopeMs = (seed: PendingTrace) =>
+ Math.max(
+ NARROW_BATCH_MAX_ENVELOPE_MS,
+ getSpanMs(seed) * (1 + WIDE_MAX_EXPANSION_RATIO),
+ );
+
+type LocalityCandidate = {
+ entry: PendingTrace;
+ minMinute: number;
+ maxMinute: number;
+ traceHash: number;
+};
+
+const toLocalityCandidate = (entry: PendingTrace): LocalityCandidate => ({
+ entry,
+ minMinute: toMinuteBucket(entry.trace.minStart),
+ maxMinute: toMinuteBucket(entry.trace.maxStart),
+ traceHash: xxh32(entry.trace.traceId),
+});
+
+type BatchSelectionCost = {
+ crossProjectBoundaries: number;
+ minuteHashCells: number;
+ traceHashGap: number;
+};
+
+const compareByClickHouseLocality = (
+ left: LocalityCandidate,
+ right: LocalityCandidate,
+) =>
+ compareStrings(left.entry.trace.projectId, right.entry.trace.projectId) ||
+ compareNumbers(left.minMinute, right.minMinute) ||
+ compareNumbers(left.maxMinute, right.maxMinute) ||
+ compareNumbers(left.traceHash, right.traceHash) ||
+ compareStrings(left.entry.trace.traceId, right.entry.trace.traceId);
+
+const compareBatchSelectionCost = (
+ left: BatchSelectionCost,
+ right: BatchSelectionCost,
+) =>
+ compareNumbers(left.crossProjectBoundaries, right.crossProjectBoundaries) ||
+ compareNumbers(left.minuteHashCells, right.minuteHashCells) ||
+ compareNumbers(left.traceHashGap, right.traceHashGap);
+
+const addBatchSelectionCost = (
+ left: BatchSelectionCost,
+ right: BatchSelectionCost,
+): BatchSelectionCost => ({
+ crossProjectBoundaries:
+ left.crossProjectBoundaries + right.crossProjectBoundaries,
+ minuteHashCells: left.minuteHashCells + right.minuteHashCells,
+ traceHashGap: left.traceHashGap + right.traceHashGap,
+});
+
+const ZERO_BATCH_SELECTION_COST: BatchSelectionCost = {
+ crossProjectBoundaries: 0,
+ minuteHashCells: 0,
+ traceHashGap: 0,
+};
+
+const selectLocalityBatches = (
+ candidates: readonly PendingTrace[],
+ maxBatchSize: number,
+): PendingTrace[][] => {
+ const sorted = candidates
+ .map(toLocalityCandidate)
+ .toSorted(compareByClickHouseLocality);
+ const candidateCount = sorted.length;
+ const stride = Math.min(maxBatchSize, candidateCount) + 1;
+ const minuteHashCellCosts = new Float64Array(candidateCount * stride);
+ const feasibleSlices = new Uint8Array(candidateCount * stride);
+ const projectBoundaryPrefix = new Uint32Array(candidateCount + 1);
+ const traceHashGapPrefix = new Float64Array(candidateCount + 1);
+
+ for (let index = 1; index < candidateCount; index++) {
+ const previous = sorted[index - 1];
+ const current = sorted[index];
+ const sameProject =
+ previous.entry.trace.projectId === current.entry.trace.projectId;
+ const sameTimeRange =
+ previous.minMinute === current.minMinute &&
+ previous.maxMinute === current.maxMinute;
+ projectBoundaryPrefix[index + 1] =
+ projectBoundaryPrefix[index] + (sameProject ? 0 : 1);
+ traceHashGapPrefix[index + 1] =
+ traceHashGapPrefix[index] +
+ (sameProject && sameTimeRange
+ ? current.traceHash - previous.traceHash
+ : 0);
+ }
+
+ for (let start = 0; start < candidateCount; start++) {
+ let minMinute = Number.POSITIVE_INFINITY;
+ let maxMinute = Number.NEGATIVE_INFINITY;
+ let minStart = Number.POSITIVE_INFINITY;
+ let maxStart = Number.NEGATIVE_INFINITY;
+ const seedEnvelopeMs = allowedEnvelopeMs(sorted[start].entry);
+ const maxLength = Math.min(maxBatchSize, candidateCount - start);
+ for (let length = 1; length <= maxLength; length++) {
+ const candidate = sorted[start + length - 1];
+ minMinute = Math.min(minMinute, candidate.minMinute);
+ maxMinute = Math.max(maxMinute, candidate.maxMinute);
+ minStart = Math.min(minStart, candidate.entry.trace.minStart);
+ maxStart = Math.max(maxStart, candidate.entry.trace.maxStart);
+ const slot = start * stride + length;
+ // The shared time predicate applies every selected trace hash across the
+ // complete envelope, so minute × hash count approximates PK cells read.
+ minuteHashCellCosts[slot] = (maxMinute - minMinute + 1) * length;
+ // Narrow groups stay within one hour. Wide groups may not grow past
+ // 125% of the first trace in locality order, the slice seed.
+ feasibleSlices[slot] = maxStart - minStart <= seedEnvelopeMs ? 1 : 0;
+ }
+ }
+
+ const minJobs = new Int32Array(candidateCount + 1).fill(-1);
+ minJobs[0] = 0;
+ for (let end = 1; end <= candidateCount; end++) {
+ const minStart = Math.max(0, end - maxBatchSize);
+ let best = Number.MAX_SAFE_INTEGER;
+ for (let start = minStart; start < end; start++) {
+ if (minJobs[start] < 0) continue;
+ if (!feasibleSlices[start * stride + (end - start)]) continue;
+ best = Math.min(best, minJobs[start] + 1);
+ }
+ if (best === Number.MAX_SAFE_INTEGER) {
+ throw new Error("Unable to partition trace batch candidates");
+ }
+ minJobs[end] = best;
+ }
+ const batchCount = minJobs[candidateCount];
+
+ const predecessors = Array.from({ length: batchCount + 1 }, () =>
+ new Int32Array(candidateCount + 1).fill(-1),
+ );
+ let previousCosts: Array<BatchSelectionCost | null> = Array(
+ candidateCount + 1,
+ ).fill(null);
+ previousCosts[0] = ZERO_BATCH_SELECTION_COST;
+
+ // Find the cheapest path through all prefixes using exactly batchCount jobs.
+ for (let batchIndex = 1; batchIndex <= batchCount; batchIndex++) {
+ const currentCosts: Array<BatchSelectionCost | null> = Array(
+ candidateCount + 1,
+ ).fill(null);
+ const remainingJobs = batchCount - batchIndex;
+ // Leave enough traces for the remaining nonempty, size-bounded batches.
+ const minEnd = Math.max(
+ batchIndex,
+ candidateCount - remainingJobs * maxBatchSize,
+ );
+ const maxEnd = Math.min(
+ candidateCount - remainingJobs,
+ batchIndex * maxBatchSize,
+ );
+ for (let end = minEnd; end <= maxEnd; end++) {
+ if (minJobs[end] > batchIndex) continue;
+ const minStart = Math.max(batchIndex - 1, end - maxBatchSize);
+ for (let start = minStart; start < end; start++) {
+ const previousCost = previousCosts[start];
+ if (!previousCost) continue;
+ const length = end - start;
+ if (!feasibleSlices[start * stride + length]) continue;
+ const batchCost: BatchSelectionCost = {
+ crossProjectBoundaries:
+ projectBoundaryPrefix[end] - projectBoundaryPrefix[start + 1],
+ minuteHashCells: minuteHashCellCosts[start * stride + length],
+ traceHashGap: traceHashGapPrefix[end] - traceHashGapPrefix[start + 1],
+ };
+ const candidateCost = addBatchSelectionCost(previousCost, batchCost);
+ const currentCost = currentCosts[end];
+ if (
+ !currentCost ||
+ compareBatchSelectionCost(candidateCost, currentCost) < 0
+ ) {
+ currentCosts[end] = candidateCost;
+ predecessors[batchIndex][end] = start;
+ }
+ }
+ }
+ previousCosts = currentCosts;
+ }
+
+ const batches = Array<PendingTrace[]>(batchCount);
+ let end = candidateCount;
+ for (let batchIndex = batchCount; batchIndex > 0; batchIndex--) {
+ const start = predecessors[batchIndex][end];
+ if (start < 0)
+ throw new Error("Unable to partition trace batch candidates");
+ batches[batchIndex - 1] = sorted
+ .slice(start, end)
+ .map(({ entry }) => entry);
+ end = start;
+ }
+ return batches;
+};
+
+/**
+ * Deterministically assigns one bounded hydrated candidate window.
+ *
+ * Locality follows the events table's physical order: project, start-time
+ * minute, and xxHash32(trace ID). A batch may not exceed one hour, or 125% of
+ * its first trace's span, whichever is larger. Extra jobs are added only when
+ * that envelope cap forbids a cheaper fill. Among feasible partitions with
+ * that minimum job count, dynamic programming first avoids crossing projects,
+ * then minimizes minute × trace-hash cells, then splits the largest hash gaps.
+ *
+ * Selection uses O(n × k × maxBatchSize) time and
+ * O(n × min(n, maxBatchSize) + n × k) memory, where k is the fewest feasible
+ * jobs (at least ceil(n / maxBatchSize), up to n for disjoint envelopes).
+ * Locality callers bound n to CHUNK_SIZE (1,000) hydrated candidates.
+ * Project mode also accepts a tail of at most maxBatchSize - 1 traces and is O(n).
+ */
+export function selectTraceBatches(
+ candidates: readonly PendingTrace[],
+ maxBatchSize: number,
+ strategy: TraceBatchStrategy,
+): PendingTrace[][] {
+ if (candidates.length === 0) return [];
+ if (strategy === "project") {
+ const batches: PendingTrace[][] = [];
+ for (let offset = 0; offset < candidates.length; offset += maxBatchSize) {
+ batches.push(candidates.slice(offset, offset + maxBatchSize));
+ }
+ return batches;
+ }
+
+ return selectLocalityBatches(candidates, maxBatchSize);
+}
+
// Retain traces for a bounded time after readiness. Reuse the due index so
// cleanup needs no full hash scan. Whole-key expiry is an inactivity backstop.
const EXPIRE_PENDING_SCRIPT = `
@@ -299,6 +544,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
// One range read captures all due IDs without cursor races or loading every
// trace's state. Memory and the Redis response size grow with the due cohort.
+ const strategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;
const members = (
await redis.zrange(
DUE_KEY,
@@ -340,10 +586,32 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
"langfuse.trace_batch.project_count",
new Set(traces.map((trace) => trace.projectId)).size,
);
+ const fill =
+ batch.length === 1
+ ? "singleton"
+ : batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE
+ ? "full"
+ : "partial";
+ recordIncrement("langfuse.trace_batch.dispatched_batches", 1, {
+ strategy,
+ fill,
+ });
+ recordDistribution(
+ "langfuse.trace_batch.event_time_envelope_ms",
+ Math.max(...traces.map(({ maxStart }) => maxStart)) -
+ Math.min(...traces.map(({ minStart }) => minStart)) +
+ 2 * QUERY_BUFFER_MS,
+ { strategy },
+ );
recordIncrement("langfuse.trace_batch.dispatched_traces", batch.length, {
batch_kind: batch.length === 1 ? "singleton" : "multi",
});
for (const entry of batch) {
+ recordDistribution(
+ "langfuse.trace_batch.observed_start_span_ms",
+ entry.trace.maxStart - entry.trace.minStart,
+ { strategy },
+ );
recordDistribution(
"langfuse.trace_batch.due_lag_ms",
Math.max(0, Date.now() - entry.due),
@@ -364,7 +632,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
);
};
- let batch: PendingTrace[] = [];
+ let projectBatchTail: PendingTrace[] = [];
for (
let offset = 0;
offset < members.length && !this.stopping;
@@ -385,11 +653,12 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
"langfuse.trace_batch.skipped_traces",
candidates.length - hydrated.length / 3,
);
+ const hydratedCandidates: PendingTrace[] = [];
for (let i = 0; i < hydrated.length; i += 3) {
if (this.stopping) return;
const member = hydrated[i];
const [projectId, traceId] = JSON.parse(member) as [string, string];
- batch.push({
+ hydratedCandidates.push({
member,
due: Number(hydrated[i + 2]),
trace: TraceBatchTraceSchema.parse({
@@ -398,12 +667,40 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
traceId,
}),
});
- if (batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {
- await enqueue(batch);
- batch = [];
+ }
+ const selectorInput =
+ strategy === "project"
+ ? [...projectBatchTail, ...hydratedCandidates]
+ : hydratedCandidates;
+ recordDistribution(
+ "langfuse.trace_batch.candidate_buffer_size",
+ selectorInput.length,
+ { strategy },
+ );
+ const selectorStartedAt = performance.now();
+ const selected = selectTraceBatches(
+ selectorInput,
+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
+ strategy,
+ );
+ recordDistribution(
+ "langfuse.trace_batch.selector_duration_ms",
+ performance.now() - selectorStartedAt,
+ { strategy },
+ );
+ if (strategy === "project") {
+ projectBatchTail = [];
+ if (selected.at(-1)?.length !== env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {
+ projectBatchTail = selected.pop() ?? [];
}
}
+ for (const batch of selected) {
+ if (this.stopping) return;
+ await enqueue(batch);
+ }
+ }
+ if (!this.stopping && projectBatchTail.length > 0) {
+ await enqueue(projectBatchTail);
}
- if (!this.stopping && batch.length > 0) await enqueue(batch);
}
}