🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): add opt-in trace batch locality selection (#17498)
Lysander Kiesele1ac230adb25
diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 992936e6de..607bf73720 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1082,6 +1082,9 @@ importers:'@mastra/mcp':specifier: 1.8.1version: 1.8.1(@cfworker/[email protected])(@mastra/[email protected](@bufbuild/[email protected])(@cfworker/[email protected])(@grpc/[email protected])([email protected]([email protected]))([email protected]([email protected]))([email protected])([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])([email protected])+ '@node-rs/xxhash':+ specifier: 1.7.7+ version: 1.7.7'@opentelemetry/api':specifier: ^1.9.1version: 1.9.1@@ -3459,6 +3462,92 @@ packages:'@nodable/[email protected]':resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==}+ '@node-rs/[email protected]':+ resolution: {integrity: sha512-9uOAXzMmayilkSmoSgrtZDKG2a4E8I0f58Ydm0/us2ubfUGqBr6AjaxzRjipuDBGeNZYYQP1wmoUzLAmuUajIQ==}+ engines: {node: '>= 12'}+ cpu: [arm]+ os: [android]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-NDqhut7aniQegfLEYBDxmhI5gFHcDur3D9G7E724SzF3aPw0lTAN9cLixd/Cs+BJRwq3oziOdGtYW9j0xM/B0Q==}+ engines: {node: '>= 12'}+ cpu: [arm64]+ os: [android]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-uwrEQELN50J2leUNBTir50nRxJJT3Dts3WDnvEij+MMJco++uI2dXoHbwP/UoavGuyoPtngrWxKP8XS9/mLvOA==}+ engines: {node: '>= 12'}+ cpu: [arm64]+ os: [darwin]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-y7AUeH573Ks5BDaJtmyvPxHhPDGedwZnU/hqFbS/DLUqpYGxXLwYkV5iSBtEfFnYyyae1MVjUBqSaCXB6wuLDg==}+ engines: {node: '>= 12'}+ cpu: [x64]+ os: [darwin]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-4kOYAbnPcFgcpOVQvthha09hjlmswIsfcZWm8uTZC1sau3tdynyyQy1/OcOG03EdP1W0qHOC3ORfUTS9oxbI9A==}+ engines: {node: '>= 12'}+ cpu: [x64]+ os: [freebsd]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-zO9DsqETMS+8/UarCugj1NW+PcD4haFfYQuVXFaPTskgtocvFY8nYY0N59YZukBWlSGh3xBtXPMVd39zeRiNKg==}+ engines: {node: '>= 12'}+ cpu: [arm]+ os: [linux]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-48kFnEIy8w1R8pGKblG+em6Q7YuDRWcLsSRpFCIrbpsA6aAQdvZnMXiyx+HSq5gEWwM35Hrb30L1sStUSetFmw==}+ engines: {node: '>= 12'}+ cpu: [arm64]+ os: [linux]+ libc: [glibc]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-m1Wz3ySODY7XZixTgLn9n59tUVSTmhO4iwcTLX5lRNnJaCOiWywsySPPbo3++oebSGTwAdJEmPJq5BLPSEZGGQ==}+ engines: {node: '>= 12'}+ cpu: [arm64]+ os: [linux]+ libc: [musl]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-rdLsrNjOUbOCdsdZZqfqeE24y/yeSR8c11w5Xkwj/Qkrcpaq3qB5f3aIW+6WuQJjdBsJwOxHCspSuUiaY6Xx4g==}+ engines: {node: '>= 12'}+ cpu: [x64]+ os: [linux]+ libc: [glibc]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-1LmaKDAIoqZcSQJG22cZMH9aL4B6jg5kQygg6pY7NyICzw+taLPLHhaMMEE+E0n2BodRXzVjZvF15pxi4qXtug==}+ engines: {node: '>= 12'}+ cpu: [x64]+ os: [linux]+ libc: [musl]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-F2QaHmplRZbg1T/AlQuW11MmRsxLvNVEsofeQd92/u704L5+BBEVc13MutJwioXNIAW1GmgOpfwVG6ZGeLAoyA==}+ engines: {node: '>= 12'}+ cpu: [arm64]+ os: [win32]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-YnHHBQVHju1Sx8vc2s2RauAGXAwWeqFMC4CvaJJ8j5WPnW2x8p9hTQRURMl/ATPLp/t8j4C9AEmLNtrNdE6JXQ==}+ engines: {node: '>= 12'}+ cpu: [ia32]+ os: [win32]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-0uVSktuJUetF467YZwHoGuBGcY8RF30DixboXTgMYG5amWze2j+OvaSpQKLsHhf0fs2UZH9W4KQG3SfUizlmGA==}+ engines: {node: '>= 12'}+ cpu: [x64]+ os: [win32]++ '@node-rs/[email protected]':+ resolution: {integrity: sha512-lQflJcUTqTvIbFNvq+6DnXQr2MCAqEBsXt9IgTiuhQeaFPejoUJBRF2WlfU5hfUzsCjU+DrzZlxzob0Q4zdHLw==}+ engines: {node: '>= 12'}+'@nodelib/[email protected]':resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}engines: {node: '>= 8'}@@ -15169,6 +15258,61 @@ snapshots:'@nodable/[email protected]': {}+ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optional: true++ '@node-rs/[email protected]':+ optionalDependencies:+ '@node-rs/xxhash-android-arm-eabi': 1.7.7+ '@node-rs/xxhash-android-arm64': 1.7.7+ '@node-rs/xxhash-darwin-arm64': 1.7.7+ '@node-rs/xxhash-darwin-x64': 1.7.7+ '@node-rs/xxhash-freebsd-x64': 1.7.7+ '@node-rs/xxhash-linux-arm-gnueabihf': 1.7.7+ '@node-rs/xxhash-linux-arm64-gnu': 1.7.7+ '@node-rs/xxhash-linux-arm64-musl': 1.7.7+ '@node-rs/xxhash-linux-x64-gnu': 1.7.7+ '@node-rs/xxhash-linux-x64-musl': 1.7.7+ '@node-rs/xxhash-win32-arm64-msvc': 1.7.7+ '@node-rs/xxhash-win32-ia32-msvc': 1.7.7+ '@node-rs/xxhash-win32-x64-msvc': 1.7.7+'@nodelib/[email protected]':dependencies:'@nodelib/fs.stat': 2.0.5diff --git a/worker/package.json b/worker/package.jsonindex 2997984744..a9c7e78d54 100644--- a/worker/package.json+++ b/worker/package.json@@ -38,6 +38,7 @@"@appsignal/opentelemetry-instrumentation-bullmq": "^0.8.0","@langfuse/native": "workspace:*","@langfuse/shared": "workspace:*",+ "@node-rs/xxhash": "1.7.7","@aws-sdk/client-lambda-microvms": "catalog:","@mastra/core": "1.46.0","@mastra/mcp": "1.8.1",diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex e52ccf7f4c..0e532336a7 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -26,6 +26,8 @@ import {import * as shared from "@langfuse/shared/src/server";import { env } from "../env";import {+ selectTraceBatches,+ type PendingTrace,trackTraceBatchActivity,TraceBatchDispatcher,} from "../features/traceBatching/traceBatching";@@ -54,6 +56,221 @@ vi.mock("../features/evaluation/observationEval", async (importOriginal) => ({fetchObservationEvalRules: vi.fn().mockResolvedValue([]),}));+describe("trace batch selection", () => {+ const minute = 60_000;+ const pendingTrace = (+ projectId: string,+ traceId: string,+ due: number,+ minStart: number,+ maxStart: number,+ ): PendingTrace => ({+ member: JSON.stringify([projectId, traceId]),+ due,+ trace: {+ projectId,+ traceId,+ minStart,+ maxStart,+ revision: `revision-${projectId}-${traceId}`,+ },+ });+ const ids = (batches: PendingTrace[][]) =>+ batches.map((batch) => batch.map(({ member }) => member));++ it("orders batches by the events table locality key", () => {+ const projectB = pendingTrace("project-b", "trace-1", 1, 0, minute);+ const laterMinute = pendingTrace(+ "project-a",+ "trace-3",+ 2,+ 2 * minute,+ 3 * minute,+ );+ const widerRange = pendingTrace(+ "project-a",+ "trace-4",+ 3,+ 10_000,+ 2 * minute,+ );+ const trace0 = pendingTrace("project-a", "trace-0", 4, 20_000, 50_000);+ const trace1 = pendingTrace("project-a", "trace-1", 5, 10_000, 50_000);++ const batches = selectTraceBatches(+ [projectB, laterMinute, widerRange, trace0, trace1],+ 2,+ "locality",+ );++ expect(ids(batches)).toEqual([+ [trace1.member, trace0.member],+ [widerRange.member, laterMinute.member],+ [projectB.member],+ ]);+ });++ it("is independent of readiness and input order", () => {+ const candidates = [+ pendingTrace("project-b", "trace-3", 1, 10 * minute, 11 * minute),+ pendingTrace("project-a", "trace-0", 4, 10 * minute, 11 * minute),+ pendingTrace("project-a", "trace-1", 3, 10 * minute, 11 * minute),+ pendingTrace("project-a", "trace-4", 2, 10 * minute, 11 * minute),+ ];+ const expected = selectTraceBatches(candidates, 3, "locality");++ expect(expected.flat().map(({ trace }) => trace.traceId)).toEqual([+ "trace-1",+ "trace-0",+ "trace-4",+ "trace-3",+ ]);+ expect(selectTraceBatches(candidates.toReversed(), 3, "locality")).toEqual(+ expected,+ );+ });++ it("moves a batch boundary instead of bridging distant time ranges", () => {+ const early = Array.from({ length: 59 }, (_, index) =>+ pendingTrace("project", `early-${index}`, index, 0, minute),+ );+ const late = Array.from({ length: 60 }, (_, index) =>+ pendingTrace(+ "project",+ `late-${index}`,+ 100 + index,+ 10_000 * minute,+ 10_001 * minute,+ ),+ );++ const batches = selectTraceBatches([...late, ...early], 60, "locality");++ expect(batches.map((batch) => batch.length)).toEqual([59, 60]);+ expect(+ batches[0].every(({ trace }) => trace.traceId.startsWith("early-")),+ ).toBe(true);+ expect(+ batches[1].every(({ trace }) => trace.traceId.startsWith("late-")),+ ).toBe(true);+ });++ it("avoids a cross-project fill when the same job count permits it", () => {+ const projectA = Array.from({ length: 59 }, (_, index) =>+ pendingTrace("project-a", `trace-${index}`, index, 0, minute),+ );+ const projectB = Array.from({ length: 60 }, (_, index) =>+ pendingTrace("project-b", `trace-${index}`, 100 + index, 0, minute),+ );++ const batches = selectTraceBatches(+ [...projectB, ...projectA],+ 60,+ "locality",+ );++ expect(batches.map((batch) => batch.length)).toEqual([59, 60]);+ expect(+ batches.map(+ (batch) => new Set(batch.map(({ trace }) => trace.projectId)).size,+ ),+ ).toEqual([1, 1]);+ });++ it("cuts at the largest trace-hash gap within one time range", () => {+ const candidates = Array.from({ length: 5 }, (_, index) =>+ pendingTrace("project", `trace-${index}`, index, 0, minute),+ );++ const batches = selectTraceBatches(candidates.toReversed(), 3, "locality");++ expect(+ batches.map((batch) => batch.map(({ trace }) => trace.traceId)),+ ).toEqual([+ ["trace-1", "trace-0"],+ ["trace-4", "trace-2", "trace-3"],+ ]);+ });++ it("splits distant narrow traces rather than sharing one envelope", () => {+ const candidates = Array.from({ length: 3 }, (_, index) =>+ pendingTrace(+ "project",+ `day-${index}`,+ index,+ index * 3 * 60 * minute,+ index * 3 * 60 * minute + minute,+ ),+ );++ expect(ids(selectTraceBatches(candidates, 60, "locality"))).toEqual(+ candidates.map((candidate) => [candidate.member]),+ );+ });++ it("preserves uneven cohorts separated by mandatory envelope boundaries", () => {+ const cohorts = [2, 1, 4, 2].map((size, cohort) =>+ Array.from({ length: size }, (_, index) => {+ const start = (cohort * 120 + index) * minute;+ return pendingTrace(+ "project",+ `cohort-${cohort}-${index}`,+ index,+ start,+ start + minute,+ );+ }),+ );++ for (const cap of [4, 9]) {+ expect(+ ids(selectTraceBatches(cohorts.flat().toReversed(), cap, "locality")),+ ).toEqual(ids(cohorts));+ }+ });++ it("bounds chained wide-trace expansion relative to the batch seed", () => {+ const seed = pendingTrace("project", "seed", 1, 0, 120 * minute);+ const compatible = pendingTrace(+ "project",+ "compatible",+ 2,+ 0,+ 144 * minute,+ );+ const wouldChain = pendingTrace(+ "project",+ "would-chain",+ 3,+ 0,+ 168 * minute,+ );++ const batches = selectTraceBatches(+ [wouldChain, compatible, seed],+ 60,+ "locality",+ );++ expect(batches.flat()).toHaveLength(3);+ expect(+ batches.some((batch) => {+ const ids = new Set(batch.map(({ member }) => member));+ return ids.has(seed.member) && ids.has(wouldChain.member);+ }),+ ).toBe(false);+ expect(+ batches.every((batch) => {+ const union =+ Math.max(...batch.map(({ trace }) => trace.maxStart)) -+ Math.min(...batch.map(({ trace }) => trace.minStart));+ const seedSpan = batch[0].trace.maxStart - batch[0].trace.minStart;+ return union <= Math.max(60 * minute, seedSpan * 1.25);+ }),+ ).toBe(true);+ });+});+describe("trace micro-batch scheduling with Redis", () => {const dueKey = "{trace-batch}:due";const stateKey = "{trace-batch}:state";@@ -62,6 +279,7 @@ describe("trace micro-batch scheduling with Redis", () => {const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;const originalSamplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;+ const originalStrategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;@@ -98,6 +316,7 @@ describe("trace micro-batch scheduling with Redis", () => {env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;+ env.LANGFUSE_TRACE_BATCH_STRATEGY = "project";env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 60;env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 7_200_000;await client().del(dueKey, stateKey, "{trace-batch}:dispatcher");@@ -120,6 +339,7 @@ describe("trace micro-batch scheduling with Redis", () => {env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = originalSamplingRate;+ env.LANGFUSE_TRACE_BATCH_STRATEGY = originalStrategy;env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;@@ -130,6 +350,70 @@ describe("trace micro-batch scheduling with Redis", () => {});afterAll(() => client().disconnect());+ it("dispatches with locality selection and records bounded selector measurements", async () => {+ env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";+ const minute = 60_000;+ await trackTraceBatchActivity("project-a", [+ event("short-1", 10 * minute),+ event("short-1", 12 * minute),+ event("wide", 0),+ event("wide", 8 * 60 * minute),+ ]);+ await trackTraceBatchActivity("project-b", [+ event("short-2", 11 * minute),+ event("short-2", 13 * minute),+ ]);+ const due = Date.now() - 1_000;+ await client().zadd(+ dueKey,+ due,+ member("project-a", "short-1"),+ due + 1,+ member("project-a", "wide"),+ due + 2,+ member("project-b", "short-2"),+ );++ await runner().processBatch();++ const jobs = await queue.getJobs(["wait"]);+ expect(+ jobs+ .map((job) =>+ job.data.payload.traces+ .map(({ traceId }) => traceId)+ .sort()+ .join(","),+ )+ .sort(),+ ).toEqual(["short-1,short-2,wide"]);+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.candidate_buffer_size",+ 3,+ { strategy: "locality" },+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.event_time_envelope_ms",+ 29_040_000,+ { strategy: "locality" },+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.dispatched_batches",+ 1,+ { strategy: "locality", fill: "partial" },+ );+ expect(+ vi+ .mocked(recordDistribution)+ .mock.calls.some(+ ([name, value, tags]) =>+ name === "langfuse.trace_batch.selector_duration_ms" &&+ Number(value) >= 0 &&+ tags?.strategy === "locality",+ ),+ ).toBe(true);+ });+it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;const evaluate = vi.spyOn(client(), "eval");@@ -500,47 +784,79 @@ describe("trace micro-batch scheduling with Redis", () => {},);- it("groups the complete due cohort by sorted project across hydration chunks and the former run limit", async () => {- const due = Date.now() - 100_000;- const traces = Array.from({ length: 10_061 }, (_, i) => ({- projectId: ["project-c", "project-a", "project-b"][i % 3],- traceId: `trace-${String(i).padStart(5, "0")}`,- due: due + Math.floor(i / 2_000) * 1_000,- }));- for (const projectId of ["project-c", "project-a", "project-b"]) {- await trackTraceBatchActivity(- projectId,- traces- .filter((trace) => trace.projectId === projectId)- .map((trace) => event(trace.traceId)),+ it.each(["project", "locality"] as const)(+ "dispatches the complete due cohort across hydration chunks with %s",+ async (strategy) => {+ env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;+ const due = Date.now() - 100_000;+ const traces = Array.from({ length: 10_061 }, (_, i) => ({+ projectId: ["project-c", "project-a", "project-b"][i % 3],+ traceId: `trace-${String(i).padStart(5, "0")}`,+ due: due + Math.floor(i / 2_000) * 1_000,+ }));+ for (const projectId of ["project-c", "project-a", "project-b"]) {+ await trackTraceBatchActivity(+ projectId,+ traces+ .filter((trace) => trace.projectId === projectId)+ .map((trace) => event(trace.traceId)),+ );+ }+ await client().zadd(+ dueKey,+ ...traces.flatMap((trace) => [+ trace.due,+ member(trace.projectId, trace.traceId),+ ]),);- }- await client().zadd(- dueKey,- ...traces.flatMap((trace) => [- trace.due,- member(trace.projectId, trace.traceId),- ]),- );- const add = vi.spyOn(queue, "add");- await runner().processBatch();- const batches = add.mock.calls.map(([, job]) => job.payload.traces);- expect(batches.map((batch) => batch.length)).toEqual([- ...Array(167).fill(60),- 41,- ]);- const expected = traces.sort(- (a, b) =>- a.projectId.localeCompare(b.projectId) ||- a.due - b.due ||- a.traceId.localeCompare(b.traceId),- );- expect(- batches.flat().map((trace) => member(trace.projectId, trace.traceId)),- ).toEqual(expected.map((trace) => member(trace.projectId, trace.traceId)));- expect(await client().zcard(dueKey)).toBe(0);- expect(await client().hlen(stateKey)).toBe(0);- });+ const add = vi.spyOn(queue, "add");+ await runner().processBatch();+ const batches = add.mock.calls.map(([, job]) => job.payload.traces);+ const assigned = batches+ .flat()+ .map((trace) => member(trace.projectId, trace.traceId));+ expect(assigned).toHaveLength(traces.length);+ expect(new Set(assigned)).toEqual(+ new Set(traces.map((trace) => member(trace.projectId, trace.traceId))),+ );+ expect(batches.every((batch) => batch.length <= 60)).toBe(true);+ const candidateSizes = vi+ .mocked(recordDistribution)+ .mock.calls.filter(+ ([name]) => name === "langfuse.trace_batch.candidate_buffer_size",+ )+ .map(([, value]) => Number(value));+ expect(Math.max(...candidateSizes)).toBeLessThanOrEqual(+ strategy === "project" ? 1059 : 1000,+ );+ if (strategy === "project") {+ expect(batches.map((batch) => batch.length)).toEqual([+ ...Array(167).fill(60),+ 41,+ ]);+ const expected = traces.sort(+ (a, b) =>+ a.projectId.localeCompare(b.projectId) ||+ a.due - b.due ||+ a.traceId.localeCompare(b.traceId),+ );+ expect(+ batches.flat().map((trace) => member(trace.projectId, trace.traceId)),+ ).toEqual(+ expected.map((trace) => member(trace.projectId, trace.traceId)),+ );+ } else {+ expect(batches.length).toBeGreaterThanOrEqual(+ Math.ceil(traces.length / 60),+ );+ expect(batches.length).toBeLessThanOrEqual(+ Math.ceil(traces.length / 1000) * Math.ceil(1000 / 60),+ );+ }+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ },+ );it("revalidates the due ID list after deletion, missing state, reactivation and new arrivals", async () => {const traces = Array.from(@@ -778,20 +1094,24 @@ describe("trace micro-batch scheduling with Redis", () => {).toBe(10_001);});- it("keeps due state on enqueue failure and retries it on the next dispatch", async () => {- await trackTraceBatchActivity("project", [event("trace")]);- await makeDue("project", "trace");- vi.spyOn(queue, "add").mockRejectedValueOnce(- new Error("enqueue unavailable"),- );- const dispatcher = runner();- await dispatcher.processBatch();- expect(await client().zcard(dueKey)).toBe(1);- expect(await client().hlen(stateKey)).toBe(1);- await dispatcher.processBatch();- expect(await queue.getWaitingCount()).toBe(1);- expect(await client().zcard(dueKey)).toBe(0);- });+ it.each(["project", "locality"] as const)(+ "keeps due state on enqueue failure and retries it with %s",+ async (strategy) => {+ env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;+ await trackTraceBatchActivity("project", [event("trace")]);+ await makeDue("project", "trace");+ vi.spyOn(queue, "add").mockRejectedValueOnce(+ new Error("enqueue unavailable"),+ );+ const dispatcher = runner();+ await dispatcher.processBatch();+ expect(await client().zcard(dueKey)).toBe(1);+ expect(await client().hlen(stateKey)).toBe(1);+ await dispatcher.processBatch();+ expect(await queue.getWaitingCount()).toBe(1);+ expect(await client().zcard(dueKey)).toBe(0);+ },+ );it.each(["updated", "deleted", "expired"] as const)("preserves arrivals during enqueue, including deleted/recreated state: %s",diff --git a/worker/src/env.ts b/worker/src/env.tsindex ac19d4a74c..5bd76cfa9a 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -132,6 +132,9 @@ const EnvSchema = z.object({.positive().max(1_000).default(60),+ LANGFUSE_TRACE_BATCH_STRATEGY: z+ .enum(["project", "locality"])+ .default("project"),LANGFUSE_TRACE_BATCH_IDLE_MS: z.coerce.number().int()diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex e8e3c23207..f1475e72eb 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -14,6 +14,7 @@ changes. These internal controls are intentionally absent from env templates.| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. || `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. || `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. || `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 1,000. || `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. || `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |@@ -67,8 +68,33 @@ window prune reads. `TRACE_QUERY_BUFFER_MS` pads the earliest and latest recordestart times by two minutes each. This fixed query margin can include observationsoutside the recorded bounds; it neither waits for arrivals nor guarantees tracecompleteness. Query limits are fixed at two threads and 30 seconds with-timeouts failing the job. This baseline has no hash-locality grouping,-per-trace windows, query tuning controls or allocation based on trace size.+timeouts failing the job. Per-trace windows, query tuning controls and allocation+based on trace size are separate experiments.++### Opt-in locality grouping++`LANGFUSE_TRACE_BATCH_STRATEGY=locality` changes grouping only after dispatch is+explicitly enabled in a cloud region. The default `project` strategy preserves+project ordering and carries unfinished jobs across hydration chunks.++Locality sorts each hydrated window by project, minimum start minute, maximum+start minute and `xxHash32(traceId)`. It finds the fewest feasible jobs under the+trace cap and an event-time envelope of at most one hour, or 125% of the first+trace's observed span if larger. Within that job count, it minimizes project+boundaries, then minute × trace count, then gaps between trace hashes. These are+read-locality proxies, not measured ClickHouse scan costs.++Locality selections dispatch immediately per window of at most 1,000 candidates.+Partials do not carry between windows, which can increase small jobs. The full+ready-ID snapshot remains unchanged; only selector input is bounded. Selector+cost is O(n × k × cap), where k is the fewest feasible jobs. Measure duration+before increasing scale, especially with distant event times that force many jobs.++Compare `dispatched_batches` (tags `strategy`, `fill`),+`event_time_envelope_ms`, `observed_start_span_ms`, `candidate_buffer_size` and+`selector_duration_ms` under `langfuse.trace_batch`. The envelope metric includes+the reader's two-minute margin on each side; the selector's envelope limit does+not. Payload and Redis state are unchanged.## Monitor dispatcher memory@@ -113,5 +139,5 @@ lazy retention rules. Drain and expired jobs are removed immediately on success.For an already enabled cloud experiment, explicitly set the new read flag to`true` when deploying this version if reads should continue; leaving it unsetdrains instead. Upgrade all producers before relying on native expiry: older-producers do not install or refresh that TTL. Locality, query controls and load+producers do not install or refresh that TTL. Locality partial carry, query controls and loadtest artifacts belong in separate follow-up changes.diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex a2dfa70155..d8af2ca339 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -1,4 +1,5 @@import { createHash, randomUUID } from "node:crypto";+import { xxh32 } from "@node-rs/xxhash";import {logger,QueueJobs,@@ -23,12 +24,256 @@ import {const DUE_KEY = "{trace-batch}:due";const STATE_KEY = "{trace-batch}:state";const CHUNK_SIZE = 1_000;-type PendingTrace = {+const QUERY_BUFFER_MS = 2 * 60_000;+const MINUTE_MS = 60_000;+const NARROW_BATCH_MAX_ENVELOPE_MS = 60 * MINUTE_MS;+const WIDE_MAX_EXPANSION_RATIO = 0.25;++export type TraceBatchStrategy = "project" | "locality";+export type PendingTrace = {member: string;due: number;trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];};+const compareNumbers = (left: number, right: number) => left - right;++const compareStrings = (left: string, right: string) =>+ left < right ? -1 : left > right ? 1 : 0;++const toMinuteBucket = (timestamp: number) => Math.floor(timestamp / MINUTE_MS);++const getSpanMs = ({ trace }: PendingTrace) => trace.maxStart - trace.minStart;++const allowedEnvelopeMs = (seed: PendingTrace) =>+ Math.max(+ NARROW_BATCH_MAX_ENVELOPE_MS,+ getSpanMs(seed) * (1 + WIDE_MAX_EXPANSION_RATIO),+ );++type LocalityCandidate = {+ entry: PendingTrace;+ minMinute: number;+ maxMinute: number;+ traceHash: number;+};++const toLocalityCandidate = (entry: PendingTrace): LocalityCandidate => ({+ entry,+ minMinute: toMinuteBucket(entry.trace.minStart),+ maxMinute: toMinuteBucket(entry.trace.maxStart),+ traceHash: xxh32(entry.trace.traceId),+});++type BatchSelectionCost = {+ crossProjectBoundaries: number;+ minuteHashCells: number;+ traceHashGap: number;+};++const compareByClickHouseLocality = (+ left: LocalityCandidate,+ right: LocalityCandidate,+) =>+ compareStrings(left.entry.trace.projectId, right.entry.trace.projectId) ||+ compareNumbers(left.minMinute, right.minMinute) ||+ compareNumbers(left.maxMinute, right.maxMinute) ||+ compareNumbers(left.traceHash, right.traceHash) ||+ compareStrings(left.entry.trace.traceId, right.entry.trace.traceId);++const compareBatchSelectionCost = (+ left: BatchSelectionCost,+ right: BatchSelectionCost,+) =>+ compareNumbers(left.crossProjectBoundaries, right.crossProjectBoundaries) ||+ compareNumbers(left.minuteHashCells, right.minuteHashCells) ||+ compareNumbers(left.traceHashGap, right.traceHashGap);++const addBatchSelectionCost = (+ left: BatchSelectionCost,+ right: BatchSelectionCost,+): BatchSelectionCost => ({+ crossProjectBoundaries:+ left.crossProjectBoundaries + right.crossProjectBoundaries,+ minuteHashCells: left.minuteHashCells + right.minuteHashCells,+ traceHashGap: left.traceHashGap + right.traceHashGap,+});++const ZERO_BATCH_SELECTION_COST: BatchSelectionCost = {+ crossProjectBoundaries: 0,+ minuteHashCells: 0,+ traceHashGap: 0,+};++const selectLocalityBatches = (+ candidates: readonly PendingTrace[],+ maxBatchSize: number,+): PendingTrace[][] => {+ const sorted = candidates+ .map(toLocalityCandidate)+ .toSorted(compareByClickHouseLocality);+ const candidateCount = sorted.length;+ const stride = Math.min(maxBatchSize, candidateCount) + 1;+ const minuteHashCellCosts = new Float64Array(candidateCount * stride);+ const feasibleSlices = new Uint8Array(candidateCount * stride);+ const projectBoundaryPrefix = new Uint32Array(candidateCount + 1);+ const traceHashGapPrefix = new Float64Array(candidateCount + 1);++ for (let index = 1; index < candidateCount; index++) {+ const previous = sorted[index - 1];+ const current = sorted[index];+ const sameProject =+ previous.entry.trace.projectId === current.entry.trace.projectId;+ const sameTimeRange =+ previous.minMinute === current.minMinute &&+ previous.maxMinute === current.maxMinute;+ projectBoundaryPrefix[index + 1] =+ projectBoundaryPrefix[index] + (sameProject ? 0 : 1);+ traceHashGapPrefix[index + 1] =+ traceHashGapPrefix[index] ++ (sameProject && sameTimeRange+ ? current.traceHash - previous.traceHash+ : 0);+ }++ for (let start = 0; start < candidateCount; start++) {+ let minMinute = Number.POSITIVE_INFINITY;+ let maxMinute = Number.NEGATIVE_INFINITY;+ let minStart = Number.POSITIVE_INFINITY;+ let maxStart = Number.NEGATIVE_INFINITY;+ const seedEnvelopeMs = allowedEnvelopeMs(sorted[start].entry);+ const maxLength = Math.min(maxBatchSize, candidateCount - start);+ for (let length = 1; length <= maxLength; length++) {+ const candidate = sorted[start + length - 1];+ minMinute = Math.min(minMinute, candidate.minMinute);+ maxMinute = Math.max(maxMinute, candidate.maxMinute);+ minStart = Math.min(minStart, candidate.entry.trace.minStart);+ maxStart = Math.max(maxStart, candidate.entry.trace.maxStart);+ const slot = start * stride + length;+ // The shared time predicate applies every selected trace hash across the+ // complete envelope, so minute × hash count approximates PK cells read.+ minuteHashCellCosts[slot] = (maxMinute - minMinute + 1) * length;+ // Narrow groups stay within one hour. Wide groups may not grow past+ // 125% of the first trace in locality order, the slice seed.+ feasibleSlices[slot] = maxStart - minStart <= seedEnvelopeMs ? 1 : 0;+ }+ }++ const minJobs = new Int32Array(candidateCount + 1).fill(-1);+ minJobs[0] = 0;+ for (let end = 1; end <= candidateCount; end++) {+ const minStart = Math.max(0, end - maxBatchSize);+ let best = Number.MAX_SAFE_INTEGER;+ for (let start = minStart; start < end; start++) {+ if (minJobs[start] < 0) continue;+ if (!feasibleSlices[start * stride + (end - start)]) continue;+ best = Math.min(best, minJobs[start] + 1);+ }+ if (best === Number.MAX_SAFE_INTEGER) {+ throw new Error("Unable to partition trace batch candidates");+ }+ minJobs[end] = best;+ }+ const batchCount = minJobs[candidateCount];++ const predecessors = Array.from({ length: batchCount + 1 }, () =>+ new Int32Array(candidateCount + 1).fill(-1),+ );+ let previousCosts: Array<BatchSelectionCost | null> = Array(+ candidateCount + 1,+ ).fill(null);+ previousCosts[0] = ZERO_BATCH_SELECTION_COST;++ // Find the cheapest path through all prefixes using exactly batchCount jobs.+ for (let batchIndex = 1; batchIndex <= batchCount; batchIndex++) {+ const currentCosts: Array<BatchSelectionCost | null> = Array(+ candidateCount + 1,+ ).fill(null);+ const remainingJobs = batchCount - batchIndex;+ // Leave enough traces for the remaining nonempty, size-bounded batches.+ const minEnd = Math.max(+ batchIndex,+ candidateCount - remainingJobs * maxBatchSize,+ );+ const maxEnd = Math.min(+ candidateCount - remainingJobs,+ batchIndex * maxBatchSize,+ );+ for (let end = minEnd; end <= maxEnd; end++) {+ if (minJobs[end] > batchIndex) continue;+ const minStart = Math.max(batchIndex - 1, end - maxBatchSize);+ for (let start = minStart; start < end; start++) {+ const previousCost = previousCosts[start];+ if (!previousCost) continue;+ const length = end - start;+ if (!feasibleSlices[start * stride + length]) continue;+ const batchCost: BatchSelectionCost = {+ crossProjectBoundaries:+ projectBoundaryPrefix[end] - projectBoundaryPrefix[start + 1],+ minuteHashCells: minuteHashCellCosts[start * stride + length],+ traceHashGap: traceHashGapPrefix[end] - traceHashGapPrefix[start + 1],+ };+ const candidateCost = addBatchSelectionCost(previousCost, batchCost);+ const currentCost = currentCosts[end];+ if (+ !currentCost ||+ compareBatchSelectionCost(candidateCost, currentCost) < 0+ ) {+ currentCosts[end] = candidateCost;+ predecessors[batchIndex][end] = start;+ }+ }+ }+ previousCosts = currentCosts;+ }++ const batches = Array<PendingTrace[]>(batchCount);+ let end = candidateCount;+ for (let batchIndex = batchCount; batchIndex > 0; batchIndex--) {+ const start = predecessors[batchIndex][end];+ if (start < 0)+ throw new Error("Unable to partition trace batch candidates");+ batches[batchIndex - 1] = sorted+ .slice(start, end)+ .map(({ entry }) => entry);+ end = start;+ }+ return batches;+};++/**+ * Deterministically assigns one bounded hydrated candidate window.+ *+ * Locality follows the events table's physical order: project, start-time+ * minute, and xxHash32(trace ID). A batch may not exceed one hour, or 125% of+ * its first trace's span, whichever is larger. Extra jobs are added only when+ * that envelope cap forbids a cheaper fill. Among feasible partitions with+ * that minimum job count, dynamic programming first avoids crossing projects,+ * then minimizes minute × trace-hash cells, then splits the largest hash gaps.+ *+ * Selection uses O(n × k × maxBatchSize) time and+ * O(n × min(n, maxBatchSize) + n × k) memory, where k is the fewest feasible+ * jobs (at least ceil(n / maxBatchSize), up to n for disjoint envelopes).+ * Locality callers bound n to CHUNK_SIZE (1,000) hydrated candidates.+ * Project mode also accepts a tail of at most maxBatchSize - 1 traces and is O(n).+ */+export function selectTraceBatches(+ candidates: readonly PendingTrace[],+ maxBatchSize: number,+ strategy: TraceBatchStrategy,+): PendingTrace[][] {+ if (candidates.length === 0) return [];+ if (strategy === "project") {+ const batches: PendingTrace[][] = [];+ for (let offset = 0; offset < candidates.length; offset += maxBatchSize) {+ batches.push(candidates.slice(offset, offset + maxBatchSize));+ }+ return batches;+ }++ return selectLocalityBatches(candidates, maxBatchSize);+}+// Retain traces for a bounded time after readiness. Reuse the due index so// cleanup needs no full hash scan. Whole-key expiry is an inactivity backstop.const EXPIRE_PENDING_SCRIPT = `@@ -299,6 +544,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {// One range read captures all due IDs without cursor races or loading every// trace's state. Memory and the Redis response size grow with the due cohort.+ const strategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;const members = (await redis.zrange(DUE_KEY,@@ -340,10 +586,32 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {"langfuse.trace_batch.project_count",new Set(traces.map((trace) => trace.projectId)).size,);+ const fill =+ batch.length === 1+ ? "singleton"+ : batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE+ ? "full"+ : "partial";+ recordIncrement("langfuse.trace_batch.dispatched_batches", 1, {+ strategy,+ fill,+ });+ recordDistribution(+ "langfuse.trace_batch.event_time_envelope_ms",+ Math.max(...traces.map(({ maxStart }) => maxStart)) -+ Math.min(...traces.map(({ minStart }) => minStart)) ++ 2 * QUERY_BUFFER_MS,+ { strategy },+ );recordIncrement("langfuse.trace_batch.dispatched_traces", batch.length, {batch_kind: batch.length === 1 ? "singleton" : "multi",});for (const entry of batch) {+ recordDistribution(+ "langfuse.trace_batch.observed_start_span_ms",+ entry.trace.maxStart - entry.trace.minStart,+ { strategy },+ );recordDistribution("langfuse.trace_batch.due_lag_ms",Math.max(0, Date.now() - entry.due),@@ -364,7 +632,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {);};- let batch: PendingTrace[] = [];+ let projectBatchTail: PendingTrace[] = [];for (let offset = 0;offset < members.length && !this.stopping;@@ -385,11 +653,12 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {"langfuse.trace_batch.skipped_traces",candidates.length - hydrated.length / 3,);+ const hydratedCandidates: PendingTrace[] = [];for (let i = 0; i < hydrated.length; i += 3) {if (this.stopping) return;const member = hydrated[i];const [projectId, traceId] = JSON.parse(member) as [string, string];- batch.push({+ hydratedCandidates.push({member,due: Number(hydrated[i + 2]),trace: TraceBatchTraceSchema.parse({@@ -398,12 +667,40 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {traceId,}),});- if (batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {- await enqueue(batch);- batch = [];+ }+ const selectorInput =+ strategy === "project"+ ? [...projectBatchTail, ...hydratedCandidates]+ : hydratedCandidates;+ recordDistribution(+ "langfuse.trace_batch.candidate_buffer_size",+ selectorInput.length,+ { strategy },+ );+ const selectorStartedAt = performance.now();+ const selected = selectTraceBatches(+ selectorInput,+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ strategy,+ );+ recordDistribution(+ "langfuse.trace_batch.selector_duration_ms",+ performance.now() - selectorStartedAt,+ { strategy },+ );+ if (strategy === "project") {+ projectBatchTail = [];+ if (selected.at(-1)?.length !== env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {+ projectBatchTail = selected.pop() ?? [];}}+ for (const batch of selected) {+ if (this.stopping) return;+ await enqueue(batch);+ }+ }+ if (!this.stopping && projectBatchTail.length > 0) {+ await enqueue(projectBatchTail);}- if (!this.stopping && batch.length > 0) await enqueue(batch);}}