🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): coalesce trace batches across hydration windows (#17500)
Lysander Kieself5c8b5d48ed5
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex 0e532336a7..d5d3a5711e 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -26,6 +26,7 @@ import {import * as shared from "@langfuse/shared/src/server";import { env } from "../env";import {+ prepareLocalityPartials,selectTraceBatches,type PendingTrace,trackTraceBatchActivity,@@ -269,6 +270,142 @@ describe("trace batch selection", () => {}),).toBe(true);});+ it("coalesces overlapping same-project partials deterministically", () => {+ const early = [+ pendingTrace("project", "early-1", 1, 0, minute),+ pendingTrace("project", "early-2", 2, minute, 2 * minute),+ ];+ const intervening = [+ pendingTrace("project", "late-1", 3, 24 * 60 * minute, 24 * 60 * minute),+ ];+ const compatible = [+ pendingTrace("project", "early-3", 4, 3 * minute, 3 * minute),+ ];++ const forward = prepareLocalityPartials(+ [early, intervening, compatible],+ 6,+ );+ const reversed = prepareLocalityPartials(+ [compatible, intervening, early],+ 6,+ );++ expect(ids(forward.carry)).toEqual(ids(reversed.carry));+ expect(ids(forward.dispatch)).toEqual(ids(reversed.dispatch));+ expect(forward.dispatch).toEqual([]);+ expect(+ forward.carry.map((batch) =>+ batch.map(({ trace }) => trace.traceId).sort(),+ ),+ ).toEqual([["early-1", "early-2", "early-3"], ["late-1"]]);+ });++ it("bounds locality partial carry without splitting batches", () => {+ const oldest = [+ pendingTrace("project", "oldest-1", 1, 0, 0),+ pendingTrace("project", "oldest-2", 2, 0, 0),+ pendingTrace("project", "oldest-3", 3, 0, 0),+ ];+ const newer = [+ pendingTrace("project", "newer-1", 4, 24 * 60 * minute, 24 * 60 * minute),+ pendingTrace("project", "newer-2", 5, 24 * 60 * minute, 24 * 60 * minute),+ pendingTrace("project", "newer-3", 6, 24 * 60 * minute, 24 * 60 * minute),+ ];++ const result = prepareLocalityPartials([newer, oldest], 6);++ expect(+ result.dispatch.map((batch) =>+ batch.map(({ trace }) => trace.traceId).sort(),+ ),+ ).toEqual([oldest.map(({ trace }) => trace.traceId)]);+ expect(+ result.carry.map((batch) =>+ batch.map(({ trace }) => trace.traceId).sort(),+ ),+ ).toEqual([newer.map(({ trace }) => trace.traceId)]);+ expect(result.carry.flat()).toHaveLength(3);+ expect(result.carry.flat().length).toBeLessThanOrEqual(5);+ });++ it("breaks equal-increase coalescing ties by canonical order", () => {+ const candidates = [+ pendingTrace("project", "trace-1", 1, 0, 0),+ pendingTrace("project", "trace-2", 1, 0, 0),+ pendingTrace("project", "trace-3", 1, 0, 0),+ ];+ const canonical = selectTraceBatches(candidates, 3, "locality")[0];++ const result = prepareLocalityPartials(+ candidates.toReversed().map((candidate) => [candidate]),+ 2,+ );++ expect(result.dispatch).toHaveLength(1);+ expect(ids(result.dispatch)).toEqual([+ canonical.slice(0, 2).map(({ member }) => member),+ ]);+ expect(ids(result.carry)).toEqual([+ canonical.slice(2).map(({ member }) => member),+ ]);+ });++ it("does not coalesce incompatible partials or exceed the trace cap", () => {+ const wide = [+ pendingTrace("project", "wide", 1, 0, 120 * minute),+ pendingTrace("project", "wide-neighbor", 2, 121 * minute, 121 * minute),+ ];+ const otherProject = [+ pendingTrace("other", "same-window", 3, 121 * minute, 121 * minute),+ ];+ const distant = [+ pendingTrace("project", "distant", 4, 24 * 60 * minute, 24 * 60 * minute),+ ];+ const atCapacity = Array.from({ length: 5 }, (_, index) =>+ pendingTrace("capacity", `trace-${index}`, 10 + index, 0, 0),+ );+ const wouldOverflow = [+ pendingTrace("capacity", "overflow", 20, minute, minute),+ ];++ const result = prepareLocalityPartials(+ [otherProject, distant, wouldOverflow, wide, atCapacity],+ 5,+ );+ const all = [...result.dispatch, ...result.carry];++ expect(+ all.some(+ (batch) =>+ batch.some(({ trace }) => trace.traceId === "wide") &&+ batch.some(({ trace }) => trace.traceId === "wide-neighbor"),+ ),+ ).toBe(true);+ expect(+ all.some(+ (batch) =>+ batch.some(({ trace }) => trace.traceId === "wide") &&+ batch.some(({ trace }) => trace.projectId === "other"),+ ),+ ).toBe(false);+ expect(+ all.some(+ (batch) =>+ batch.some(({ trace }) => trace.traceId === "wide") &&+ batch.some(({ trace }) => trace.traceId === "distant"),+ ),+ ).toBe(false);+ expect(+ all.some(+ (batch) =>+ batch.some(({ trace }) => trace.traceId === "overflow") &&+ batch.some(({ trace }) => trace.traceId === "trace-0"),+ ),+ ).toBe(false);+ expect(all.every((batch) => batch.length <= 5)).toBe(true);+ expect(new Set(all.flat().map(({ member }) => member)).size).toBe(10);+ });});describe("trace micro-batch scheduling with Redis", () => {@@ -826,9 +963,7 @@ describe("trace micro-batch scheduling with Redis", () => {([name]) => name === "langfuse.trace_batch.candidate_buffer_size",).map(([, value]) => Number(value));- expect(Math.max(...candidateSizes)).toBeLessThanOrEqual(- strategy === "project" ? 1059 : 1000,- );+ expect(Math.max(...candidateSizes)).toBeLessThanOrEqual(1059);if (strategy === "project") {expect(batches.map((batch) => batch.length)).toEqual([...Array(167).fill(60),@@ -858,6 +993,165 @@ describe("trace micro-batch scheduling with Redis", () => {},);+ it("keeps project locality and a bounded partial tail across hydration chunks", async () => {+ env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";+ const traces = Array.from({ length: 2_001 }, (_, index) => ({+ projectId: index % 2 === 0 ? "project-a" : "project-b",+ traceId: `trace-${String(index).padStart(4, "0")}`,+ }));+ for (const projectId of ["project-a", "project-b"]) {+ await trackTraceBatchActivity(+ projectId,+ traces+ .filter((trace) => trace.projectId === projectId)+ .map((trace) => event(trace.traceId)),+ );+ }+ const due = Date.now() - 1_000;+ await client().zadd(+ dueKey,+ ...traces.flatMap((trace) => [+ due,+ member(trace.projectId, trace.traceId),+ ]),+ );+ const add = vi.spyOn(queue, "add");++ await runner().processBatch();++ const batches = add.mock.calls.map(([, job]) => job.payload.traces);+ expect(batches.length).toBeGreaterThanOrEqual(+ Math.ceil(traces.length / 60),+ );+ expect(batches.length).toBeLessThanOrEqual(2 * Math.ceil(1_000 / 60) + 1);+ expect(+ batches.every((batch) => batch.length > 0 && batch.length <= 60),+ ).toBe(true);+ expect(+ batches.filter(+ (batch) => new Set(batch.map((trace) => trace.projectId)).size > 1,+ ),+ ).toHaveLength(0);+ const dispatchedMembers = batches+ .flat()+ .map((trace) => member(trace.projectId, trace.traceId));+ expect(dispatchedMembers).toHaveLength(traces.length);+ expect(new Set(dispatchedMembers)).toEqual(+ new Set(traces.map((trace) => member(trace.projectId, trace.traceId))),+ );+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(+ ([name]) => name === "langfuse.trace_batch.skipped_traces",+ )+ .map(([, value]) => value),+ ).toEqual([0, 0, 0]);+ expect(await queue.getWaitingCount()).toBe(batches.length);+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ });++ it("coalesces a non-final compatible partial across a hydration boundary", async () => {+ env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";+ const minute = 60_000;+ const early = Array.from({ length: 30 }, (_, index) => ({+ traceId: `early-${String(index).padStart(2, "0")}`,+ start: 10 * minute,+ }));+ const filler = Array.from({ length: 960 }, (_, index) => ({+ traceId: `filler-${String(index).padStart(3, "0")}`,+ start: 12 * 60 * minute,+ }));+ const late = Array.from({ length: 12 }, (_, index) => ({+ traceId: `late-${String(index).padStart(2, "0")}`,+ start: 24 * 60 * minute,+ }));+ const firstWindow = [...early.slice(0, 28), ...filler, ...late];+ const traces = [...firstWindow, ...early.slice(28)];++ await trackTraceBatchActivity(+ "project",+ traces.map(({ traceId, start }) => event(traceId, start)),+ );+ const due = Date.now() - 10_000;+ await client().zadd(+ dueKey,+ ...traces.flatMap(({ traceId }, index) => [+ due + index,+ member("project", traceId),+ ]),+ );+ const targetMember = member("project", early[0].traceId);+ const hydratedRevision = JSON.parse(+ (await client().hget(stateKey, targetMember))!,+ ).revision;+ const evaluate = client().eval.bind(client());+ let reactivated = false;+ vi.spyOn(client(), "eval").mockImplementation(async (...args) => {+ const result = await evaluate(...args);+ if (+ !reactivated &&+ typeof args[0] === "string" &&+ args[0].includes("local result = {}")+ ) {+ reactivated = true;+ await trackTraceBatchActivity("project", [+ event(early[0].traceId, 11 * minute),+ ]);+ }+ return result;+ });+ const add = vi.spyOn(queue, "add");++ await runner().processBatch();++ expect(reactivated).toBe(true);+ const batches = add.mock.calls.map(([, job]) => job.payload.traces);+ expect(+ batches+ .filter((batch) =>+ batch.some(({ traceId }) => traceId.startsWith("early-")),+ )+ .map((batch) => batch.map(({ traceId }) => traceId).sort()),+ ).toEqual([early.map(({ traceId }) => traceId)]);+ expect(batches.map((batch) => batch.length).sort((a, b) => a - b)).toEqual([+ 12,+ 30,+ ...Array(16).fill(60),+ ]);+ const assigned = batches+ .flatMap((batch) => batch)+ .map(({ projectId, traceId }) => member(projectId, traceId));+ expect(assigned).toHaveLength(traces.length);+ expect(new Set(assigned)).toEqual(+ new Set(traces.map(({ traceId }) => member("project", traceId))),+ );+ const candidateBufferSizes = vi+ .mocked(recordDistribution)+ .mock.calls.filter(+ ([name]) => name === "langfuse.trace_batch.candidate_buffer_size",+ )+ .map(([, value]) => Number(value));+ expect(candidateBufferSizes).toContain(42);+ expect(Math.max(...candidateBufferSizes)).toBeLessThanOrEqual(1_059);+ const current = JSON.parse((await client().hget(stateKey, targetMember))!);+ const dispatched = batches+ .flat()+ .find(({ traceId }) => traceId === early[0].traceId);+ expect(dispatched?.revision).toBe(hydratedRevision);+ expect(current.revision).not.toBe(hydratedRevision);+ expect(current.maxStart).toBe(11 * minute);+ expect(await client().zcard(dueKey)).toBe(1);+ expect(await client().hlen(stateKey)).toBe(1);++ await makeDue("project", early[0].traceId);+ await runner().processBatch();+ expect(await queue.getWaitingCount()).toBe(batches.length + 1);+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ });+it("revalidates the due ID list after deletion, missing state, reactivation and new arrivals", async () => {const traces = Array.from({ length: 2_001 },diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex f1475e72eb..5462ff206e 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -84,9 +84,14 @@ trace's observed span if larger. Within that job count, it minimizes projectboundaries, then minute × trace count, then gaps between trace hashes. These areread-locality proxies, not measured ClickHouse scan costs.-Locality selections dispatch immediately per window of at most 1,000 candidates.-Partials do not carry between windows, which can increase small jobs. The full-ready-ID snapshot remains unchanged; only selector input is bounded. Selector+Locality combines each window of at most 1,000 hydrated candidates with retained+partials. Compatible same-project partials may merge when their buffered time+intervals overlap and the combined batch respects the cap and envelope limits.+At most `maxBatchSize - 1` traces are retained across all partials; oldest-due+partials dispatch first when that budget is exceeded. The last window flushes+all remaining partials, so carry never outlives the current dispatch run.++The full ready-ID snapshot remains unchanged; only selector input is bounded. Selectorcost is O(n × k × cap), where k is the fewest feasible jobs. Measure durationbefore increasing scale, especially with distant event times that force many jobs.@@ -139,5 +144,5 @@ lazy retention rules. Drain and expired jobs are removed immediately on success.For an already enabled cloud experiment, explicitly set the new read flag to`true` when deploying this version if reads should continue; leaving it unsetdrains instead. Upgrade all producers before relying on native expiry: older-producers do not install or refresh that TTL. Locality partial carry, query controls and load+producers do not install or refresh that TTL. Query controls and loadtest artifacts belong in separate follow-up changes.diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex d8af2ca339..7c3588cc21 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -81,6 +81,46 @@ const compareByClickHouseLocality = (compareNumbers(left.traceHash, right.traceHash) ||compareStrings(left.entry.trace.traceId, right.entry.trace.traceId);+const canonicalizeLocalityBatch = (+ batch: readonly PendingTrace[],+): PendingTrace[] =>+ batch+ .map(toLocalityCandidate)+ .toSorted(compareByClickHouseLocality)+ .map(({ entry }) => entry);++const compareCanonicalBatches = (+ left: readonly PendingTrace[],+ right: readonly PendingTrace[],+) => {+ const leftCandidates = left.map(toLocalityCandidate);+ const rightCandidates = right.map(toLocalityCandidate);+ const sharedLength = Math.min(leftCandidates.length, rightCandidates.length);+ for (let index = 0; index < sharedLength; index++) {+ const comparison = compareByClickHouseLocality(+ leftCandidates[index],+ rightCandidates[index],+ );+ if (comparison !== 0) return comparison;+ }+ return compareNumbers(leftCandidates.length, rightCandidates.length);+};++const getBatchBounds = (batch: readonly PendingTrace[]) => ({+ minStart: Math.min(...batch.map(({ trace }) => trace.minStart)),+ maxStart: Math.max(...batch.map(({ trace }) => trace.maxStart)),+});++const isFeasibleLocalityBatch = (+ batch: readonly PendingTrace[],+ maxBatchSize: number,+) => {+ if (batch.length === 0 || batch.length > maxBatchSize) return false;+ const canonical = canonicalizeLocalityBatch(batch);+ const { minStart, maxStart } = getBatchBounds(canonical);+ return maxStart - minStart <= allowedEnvelopeMs(canonical[0]);+};+const compareBatchSelectionCost = (left: BatchSelectionCost,right: BatchSelectionCost,@@ -254,7 +294,8 @@ const selectLocalityBatches = (* Selection uses O(n × k × maxBatchSize) time and* O(n × min(n, maxBatchSize) + n × k) memory, where k is the fewest feasible* jobs (at least ceil(n / maxBatchSize), up to n for disjoint envelopes).- * Locality callers bound n to CHUNK_SIZE (1,000) hydrated candidates.+ * Locality callers bound n to CHUNK_SIZE (1,000) hydrated candidates plus+ * at most maxBatchSize - 1 traces across all retained partials.* Project mode also accepts a tail of at most maxBatchSize - 1 traces and is O(n).*/export function selectTraceBatches(@@ -274,6 +315,124 @@ export function selectTraceBatches(return selectLocalityBatches(candidates, maxBatchSize);}+const compareBatchesByOldestDue = (+ left: readonly PendingTrace[],+ right: readonly PendingTrace[],+) =>+ compareNumbers(+ Math.min(...left.map(({ due }) => due)),+ Math.min(...right.map(({ due }) => due)),+ ) || compareCanonicalBatches(left, right);++const canCoalesceLocalityPartials = (+ left: readonly PendingTrace[],+ right: readonly PendingTrace[],+ maxBatchSize: number,+) => {+ if (left.length + right.length > maxBatchSize) return false;+ const projectId = left[0]?.trace.projectId;+ if (+ !projectId ||+ left.some(({ trace }) => trace.projectId !== projectId) ||+ right.some(({ trace }) => trace.projectId !== projectId)+ ) {+ return false;+ }+ const leftBounds = getBatchBounds(left);+ const rightBounds = getBatchBounds(right);+ const bufferedIntervalsOverlap =+ leftBounds.minStart - QUERY_BUFFER_MS <=+ rightBounds.maxStart + QUERY_BUFFER_MS &&+ rightBounds.minStart - QUERY_BUFFER_MS <=+ leftBounds.maxStart + QUERY_BUFFER_MS;+ return (+ bufferedIntervalsOverlap &&+ isFeasibleLocalityBatch([...left, ...right], maxBatchSize)+ );+};++/**+ * Coalesce compatible locality partials and retain a globally bounded carry.+ * The dispatcher calls this once per hydrated window; work never survives the+ * current dispatch run.+ */+export function prepareLocalityPartials(+ partials: readonly (readonly PendingTrace[])[],+ maxBatchSize: number,+): { dispatch: PendingTrace[][]; carry: PendingTrace[][] } {+ const batches = partials+ .filter((batch) => batch.length > 0)+ .map(canonicalizeLocalityBatch)+ .toSorted(compareCanonicalBatches);++ while (true) {+ let best:+ | {+ leftIndex: number;+ rightIndex: number;+ envelopeIncrease: number;+ merged: PendingTrace[];+ }+ | undefined;+ for (let leftIndex = 0; leftIndex < batches.length; leftIndex++) {+ for (+ let rightIndex = leftIndex + 1;+ rightIndex < batches.length;+ rightIndex+++ ) {+ const left = batches[leftIndex];+ const right = batches[rightIndex];+ if (!canCoalesceLocalityPartials(left, right, maxBatchSize)) continue;+ const leftBounds = getBatchBounds(left);+ const rightBounds = getBatchBounds(right);+ const merged = canonicalizeLocalityBatch([...left, ...right]);+ const mergedBounds = getBatchBounds(merged);+ const envelopeIncrease =+ mergedBounds.maxStart -+ mergedBounds.minStart -+ Math.max(+ leftBounds.maxStart - leftBounds.minStart,+ rightBounds.maxStart - rightBounds.minStart,+ );+ if (+ !best ||+ envelopeIncrease < best.envelopeIncrease ||+ (envelopeIncrease === best.envelopeIncrease &&+ compareCanonicalBatches(merged, best.merged) < 0)+ ) {+ best = { leftIndex, rightIndex, envelopeIncrease, merged };+ }+ }+ }+ if (!best) break;+ batches[best.leftIndex] = best.merged;+ batches.splice(best.rightIndex, 1);+ batches.sort(compareCanonicalBatches);+ }++ const dispatch = batches+ .filter((batch) => batch.length === maxBatchSize)+ .toSorted(compareBatchesByOldestDue);+ const carry = batches+ .filter((batch) => batch.length < maxBatchSize)+ .toSorted(compareCanonicalBatches);+ const carryBudget = maxBatchSize - 1;+ let carriedTraceCount = carry.reduce(+ (count, batch) => count + batch.length,+ 0,+ );+ while (carriedTraceCount > carryBudget) {+ carry.sort(compareBatchesByOldestDue);+ const batch = carry.shift();+ if (!batch) break;+ dispatch.push(batch);+ carriedTraceCount -= batch.length;+ }+ dispatch.sort(compareBatchesByOldestDue);+ carry.sort(compareCanonicalBatches);+ return { dispatch, carry };+}+// Retain traces for a bounded time after readiness. Reuse the due index so// cleanup needs no full hash scan. Whole-key expiry is an inactivity backstop.const EXPIRE_PENDING_SCRIPT = `@@ -633,6 +792,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {};let projectBatchTail: PendingTrace[] = [];+ let localityPartials: PendingTrace[][] = [];for (let offset = 0;offset < members.length && !this.stopping;@@ -668,13 +828,18 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {}),});}++ const carriedTraceCount =+ strategy === "project"+ ? projectBatchTail.length+ : localityPartials.reduce((count, batch) => count + batch.length, 0);const selectorInput =strategy === "project"? [...projectBatchTail, ...hydratedCandidates]- : hydratedCandidates;+ : [...localityPartials.flat(), ...hydratedCandidates];recordDistribution("langfuse.trace_batch.candidate_buffer_size",- selectorInput.length,+ hydratedCandidates.length + carriedTraceCount,{ strategy },);const selectorStartedAt = performance.now();@@ -688,19 +853,49 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {performance.now() - selectorStartedAt,{ strategy },);+if (strategy === "project") {projectBatchTail = [];if (selected.at(-1)?.length !== env.LANGFUSE_TRACE_BATCH_MAX_SIZE) {projectBatchTail = selected.pop() ?? [];}+ for (const batch of selected) {+ if (this.stopping) return;+ await enqueue(batch);+ }+ continue;}- for (const batch of selected) {++ const fullBatches = selected.filter(+ (batch) => batch.length === env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ );+ const partialBatches = selected.filter(+ (batch) => batch.length < env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ );+ for (const batch of fullBatches) {+ if (this.stopping) return;+ await enqueue(batch);+ }+ await this.extendLockOnProgress();+ if (this.stopping) return;+ const prepared = prepareLocalityPartials(+ partialBatches,+ env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ );+ localityPartials = prepared.carry;+ for (const batch of prepared.dispatch) {if (this.stopping) return;await enqueue(batch);}}- if (!this.stopping && projectBatchTail.length > 0) {- await enqueue(projectBatchTail);+ if (this.stopping) return;+ if (strategy === "project") {+ if (projectBatchTail.length > 0) await enqueue(projectBatchTail);+ return;+ }+ for (const batch of localityPartials.toSorted(compareBatchesByOldestDue)) {+ if (this.stopping) return;+ await enqueue(batch);}}}