Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(trace-batch): bound batches by estimated weight and exclude oversized traces (#18296)

Lysander Kieselc49954f1caf8
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
index 4fe9f45858..49841ff91d 100644
--- a/worker/src/__tests__/traceBatching.test.ts
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -410,6 +410,111 @@ describe("trace batch selection", () => {
expect(all.every((batch) => batch.length <= 5)).toBe(true);
expect(new Set(all.flat().map(({ member }) => member)).size).toBe(10);
});
+
+ describe("weight limits", () => {
+ const weighted = (
+ traceId: string,
+ eventUpdateCount: number,
+ serializedEventBytes = eventUpdateCount * 1_000,
+ ): PendingTrace => ({
+ ...pendingTrace("project", traceId, 1, 0, minute),
+ estimates: { eventUpdateCount, serializedEventBytes },
+ });
+ const limits = { maxEventUpdates: 10, maxSerializedEventBytes: 50_000 };
+ const weightOf = (batch: PendingTrace[]) =>
+ batch.reduce(
+ (total, { estimates }) => ({
+ events: total.events + (estimates?.eventUpdateCount ?? 0),
+ bytes: total.bytes + (estimates?.serializedEventBytes ?? 0),
+ }),
+ { events: 0, bytes: 0 },
+ );
+
+ it.each(["project", "locality"] as const)(
+ "closes %s batches before the event or byte budget and isolates overweight traces",
+ (strategy) => {
+ const candidates = [
+ weighted("a", 4),
+ weighted("b", 4),
+ weighted("c", 4),
+ weighted("heavy", 25),
+ weighted("bytes-1", 1, 30_000),
+ weighted("bytes-2", 1, 30_000),
+ { ...pendingTrace("project", "unknown", 1, 0, minute) },
+ ];
+
+ const batches = selectTraceBatches(candidates, 60, strategy, limits);
+
+ expect(batches.flat()).toHaveLength(candidates.length);
+ for (const batch of batches) {
+ if (batch.length === 1) continue;
+ const { events, bytes } = weightOf(batch);
+ expect(events).toBeLessThanOrEqual(limits.maxEventUpdates);
+ expect(bytes).toBeLessThanOrEqual(limits.maxSerializedEventBytes);
+ }
+ expect(
+ batches.find((batch) =>
+ batch.some(({ trace }) => trace.traceId === "heavy"),
+ ),
+ ).toHaveLength(1);
+ expect(
+ batches.some(
+ (batch) =>
+ batch.some(({ trace }) => trace.traceId === "bytes-1") &&
+ batch.some(({ trace }) => trace.traceId === "bytes-2"),
+ ),
+ ).toBe(false);
+ // Without budgets the same candidates fit one batch.
+ expect(selectTraceBatches(candidates, 60, strategy)).toHaveLength(1);
+ },
+ );
+
+ it("packs light locality neighbors of a heavy trace into full batches", () => {
+ const at = (
+ traceId: string,
+ start: number,
+ eventUpdateCount: number,
+ ): PendingTrace => ({
+ ...pendingTrace("project", traceId, 1, start, start),
+ estimates: {
+ eventUpdateCount,
+ serializedEventBytes: eventUpdateCount * 1_000,
+ },
+ });
+ // Locality order: two light traces, the heavy trace, six light traces.
+ const candidates = [
+ at("light-0", 0, 1),
+ at("light-1", 0, 1),
+ at("heavy", minute, 9),
+ ...Array.from({ length: 6 }, (_, index) =>
+ at(`light-${index + 2}`, 2 * minute, 1),
+ ),
+ ];
+
+ const batches = selectTraceBatches(candidates, 4, "locality", limits);
+
+ expect(batches.flat()).toHaveLength(candidates.length);
+ expect(batches).toHaveLength(3);
+ expect(
+ batches.find((batch) =>
+ batch.some(({ trace }) => trace.traceId === "heavy"),
+ ),
+ ).toHaveLength(1);
+ });
+
+ it("does not coalesce locality partials past the weight budget", () => {
+ const result = prepareLocalityPartials(
+ [[weighted("a", 6)], [weighted("b", 6)], [weighted("c", 4)]],
+ 60,
+ limits,
+ );
+ const all = [...result.dispatch, ...result.carry];
+
+ expect(all.flat()).toHaveLength(3);
+ expect(all.every((batch) => weightOf(batch).events <= 10)).toBe(true);
+ expect(all).toHaveLength(2);
+ });
+ });
});
describe("trace micro-batch scheduling with Redis", () => {
@@ -426,6 +531,10 @@ describe("trace micro-batch scheduling with Redis", () => {
const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;
const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;
const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;
+ const originalMaxTraceEventUpdates =
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES;
+ const originalMaxTraceBytes =
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES;
let queue: Queue<TQueueJobTypes[QueueName.TraceBatch]>;
let connection: NonNullable<ReturnType<typeof createNewRedisInstance>>;
const runners: TraceBatchDispatcher[] = [];
@@ -493,6 +602,9 @@ describe("trace micro-batch scheduling with Redis", () => {
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;
env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES =
+ originalMaxTraceEventUpdates;
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES = originalMaxTraceBytes;
await queue.obliterate({ force: true });
await queue.close();
connection.disconnect();
@@ -662,6 +774,93 @@ describe("trace micro-batch scheduling with Redis", () => {
).toBe(true);
});
+ it("excludes oversized traces from dispatch, removes their state, and reports their IDs", async () => {
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES = 3;
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES = 1_000;
+ const warn = vi.spyOn(shared.logger, "warn");
+ await trackTraceBatchActivity("project", [
+ ...Array.from({ length: 4 }, (_, index) =>
+ event("many-updates", 1_000_000 + index),
+ ),
+ event("large-input", 1_000_000, 5_000),
+ event("small", 1_000_000),
+ ]);
+ await makeDue("project", "many-updates", "large-input", "small");
+
+ await runner().processBatch();
+
+ const jobs = await queue.getJobs(["wait"]);
+ expect(
+ jobs.map((job) => job.data.payload.traces.map(({ traceId }) => traceId)),
+ ).toEqual([["small"]]);
+ expect(await client().zcard(dueKey)).toBe(0);
+ expect(await client().hlen(stateKey)).toBe(0);
+ for (const [traceId, reason] of [
+ ["many-updates", "event_updates"],
+ ["large-input", "serialized_bytes"],
+ ] as const) {
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.excluded_traces",
+ 1,
+ { reason, strategy: "project" },
+ );
+ expect(warn).toHaveBeenCalledWith(
+ "Trace batch excluded oversized trace",
+ expect.objectContaining({ projectId: "project", traceId, reason }),
+ );
+ }
+ });
+
+ it("does not report an oversized trace as excluded when new activity keeps it pending", async () => {
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES = 3;
+ const warn = vi.spyOn(shared.logger, "warn");
+ await trackTraceBatchActivity(
+ "project",
+ Array.from({ length: 4 }, (_, index) =>
+ event("reactivated", 1_000_000 + index),
+ ),
+ );
+ await makeDue("project", "reactivated");
+ const evaluate = client().eval.bind(client());
+ let reactivated = false;
+ vi.spyOn(client(), "eval").mockImplementation((async (
+ script: string,
+ ...args: (string | number)[]
+ ) => {
+ if (!reactivated && script.includes("cjson.decode(state).revision ==")) {
+ reactivated = true;
+ const traceMember = member("project", "reactivated");
+ const state = JSON.parse((await client().hget(stateKey, traceMember))!);
+ await client().hset(
+ stateKey,
+ traceMember,
+ JSON.stringify({ ...state, revision: randomUUID() }),
+ );
+ }
+ return evaluate(script, ...args);
+ }) as typeof evaluate);
+
+ await runner().processBatch();
+
+ expect(reactivated).toBe(true);
+ expect(
+ await client().hexists(stateKey, member("project", "reactivated")),
+ ).toBe(1);
+ expect(recordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.excluded_traces",
+ expect.anything(),
+ expect.anything(),
+ );
+ expect(warn).not.toHaveBeenCalledWith(
+ "Trace batch excluded oversized trace",
+ expect.anything(),
+ );
+ expect(recordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.reactivated_traces",
+ 1,
+ );
+ });
+
it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
const evaluate = vi.spyOn(client(), "eval");
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 288f17a32a..f7a7cce803 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -151,6 +151,27 @@ const EnvSchema = z.object({
.positive()
.max(10_000)
.default(60),
+ // Weight budgets use ingestion-time estimates; 0 disables a budget.
+ LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES: z.coerce
+ .number()
+ .int()
+ .nonnegative()
+ .default(1_500),
+ LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES: z.coerce
+ .number()
+ .int()
+ .nonnegative()
+ .default(64 * 1024 * 1024),
+ LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES: z.coerce
+ .number()
+ .int()
+ .nonnegative()
+ .default(5_000),
+ LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES: z.coerce
+ .number()
+ .int()
+ .nonnegative()
+ .default(50 * 1024 * 1024),
LANGFUSE_TRACE_BATCH_STRATEGY: z
.enum(["project", "locality"])
.default("project"),
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index 2bc8324c31..94d9dd97da 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -9,23 +9,27 @@ absent from env templates.
## Controls
-| Variable | Default | Role |
-| ------------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------- |
-| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
-| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
-| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
-| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
-| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |
-| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |
-| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
-| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |
-| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |
-| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
-| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
-| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |
-| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
-| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
+| Variable | Default | Role |
+| ------------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------- |
+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
+| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |
+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |
+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |
+| `LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES` | `1500` | Estimated event updates per multi-trace job; `0` disables. |
+| `LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES` | `67108864` | Estimated serialized event bytes per multi-trace job (64 MiB); `0` disables. |
+| `LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` | `5000` | Exclude a trace whose pending window exceeds this many event updates; `0` disables. |
+| `LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` | `52428800` | Exclude a trace whose pending window exceeds these serialized bytes (50 MiB); `0` disables. |
+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |
+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |
+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
Producer, dispatcher and consumer can run independently. Sampling reuses the
evaluator's deterministic trace-ID sampler; every observation of the same trace
@@ -57,7 +61,7 @@ than retention; ingestion and dispatcher both perform cleanup.
One dispatcher holds the renewable lease. It snapshots ready IDs, sorts them by
project, then hydrates and revalidates state in chunks of 1,000. Jobs contain up
-to the configured trace cap; project leftovers can share a job. The dispatcher
+to the configured trace cap and weight budgets; project leftovers can share a job. The dispatcher
enqueues before acknowledging matching revisions, so concurrent intake stays
pending and failed enqueueing can be retried. Stable job IDs limit duplicates.
@@ -76,17 +80,39 @@ The dispatcher emits `estimated_event_update_count` and
`estimated_serialized_event_bytes` distributions under `langfuse.trace_batch`, tagged
with `scope:trace|batch` and `strategy`. Batch totals require known estimates for
every member; `estimates_unavailable` counts unknown traces and affected batches.
-Estimates stay out of queue payloads, job IDs and batch selection. They do not cap
-work or change oversized-trace handling. Dispatch retries can emit another sample.
+Estimates stay out of queue payloads and job IDs. Dispatch retries can emit
+another sample.
Serialized event bytes differ from the reader's `input_bytes`, `output_bytes`
and `metadata_bytes` metrics: they include the serialized event fields and JSON
encoding, and do not represent RAM, network transfer or ClickHouse scan bytes.
The job return still sums those three as `ioMetadataBytes`; that sum is not a
-custom metric. After rollout, collect a stable hour
-with unchanged sampling and batching settings; compare trace/batch distributions,
-unknown-estimate counts, ingestion latency and Redis command rate/CPU/memory with
-the baseline before choosing any size-aware batching policy.
+custom metric.
+
+### Weight budgets and oversized traces
+
+Both strategies close a job before its summed estimates would exceed
+`LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES` or
+`LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES`, in addition to the trace cap.
+Locality treats an over-budget slice like one past its time envelope, and
+partials coalesce only within the budget. A single trace always forms a job, so
+a trace heavier than the budget runs alone. Locality packs traces above half of
+either budget in a separate lane, so one heavy trace does not split the light
+traces around it into extra jobs. Unknown estimates weigh nothing.
+Budget-limited jobs report `fill:partial`; the `scope:batch` estimate
+distributions show how close jobs run to the budget.
+
+After hydration, a trace whose pending-window estimates exceed
+`LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` or
+`LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` is never queued or read. The
+dispatcher acknowledges it like a dispatched trace, increments
+`langfuse.trace_batch.excluded_traces` tagged with
+`reason:event_updates|serialized_bytes` and `strategy`, and logs
+`Trace batch excluded oversized trace` at warn level with `projectId`, `traceId`,
+`reason` and both estimates. Group that log by `@projectId` to find repeat
+offenders. Activity after the acknowledgement starts a new pending window that
+is evaluated again, so a trace that keeps growing may be logged more than once.
+Traces with unknown estimates are never excluded.
The worker streams full `events_full` payloads and records batch observation,
trace and logical payload-byte metrics. The `input_bytes`, `output_bytes`
@@ -105,8 +131,9 @@ This fixed margin neither waits for arrivals nor guarantees trace completeness;
repeated intervals for the same pair form a union without duplicating rows.
Queries use response compression and chunked multipart parameters for up to
10,000 UUID-sized trace/project IDs. The fixed 30-second timeout fails the job
-rather than accepting partial results. This does not cap observation count,
-query memory or dispatcher snapshot memory.
+rather than accepting partial results. Beyond the dispatcher's weight budgets
+and trace exclusion, this does not cap observation count, query memory or
+dispatcher snapshot memory.
## Per-trace transcripts
@@ -457,8 +484,8 @@ project ordering and carries unfinished jobs across hydration chunks.
Locality sorts each hydrated window by project, minimum start minute, maximum
start minute and `xxHash32(traceId)`. It finds the fewest feasible jobs under the
-trace cap and an event-time envelope of at most one hour, or 125% of the first
-trace's observed span if larger. Within that job count, it minimizes project
+trace cap, the weight budgets and an event-time envelope of at most one hour, or
+125% of the first trace's observed span if larger. Within that job count, it minimizes project
boundaries, then minute × trace count, then gaps between trace hashes. These are
read-locality proxies, not measured ClickHouse scan costs.
diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.ts
index 73f1f81e6b..5bb400ca01 100644
--- a/worker/src/features/traceBatching/traceBatching.ts
+++ b/worker/src/features/traceBatching/traceBatching.ts
@@ -37,6 +37,68 @@ export type PendingTrace = {
trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];
};
+/**
+ * Upper bounds on the summed ingestion estimates of one dispatched batch.
+ * Traces without estimates weigh nothing, and a single trace always forms a
+ * batch even when it alone exceeds a budget.
+ */
+export type TraceBatchWeightLimits = {
+ maxEventUpdates: number;
+ maxSerializedEventBytes: number;
+};
+
+const UNBOUNDED_WEIGHT_LIMITS: TraceBatchWeightLimits = {
+ maxEventUpdates: Number.POSITIVE_INFINITY,
+ maxSerializedEventBytes: Number.POSITIVE_INFINITY,
+};
+
+const toLimit = (value: number) =>
+ value > 0 ? value : Number.POSITIVE_INFINITY;
+
+const getEventUpdates = ({ estimates }: PendingTrace) =>
+ estimates?.eventUpdateCount ?? 0;
+
+const getSerializedEventBytes = ({ estimates }: PendingTrace) =>
+ estimates?.serializedEventBytes ?? 0;
+
+const fitsWeightLimits = (
+ eventUpdates: number,
+ serializedEventBytes: number,
+ limits: TraceBatchWeightLimits,
+) =>
+ eventUpdates <= limits.maxEventUpdates &&
+ serializedEventBytes <= limits.maxSerializedEventBytes;
+
+const isWithinWeightLimits = (
+ batch: readonly PendingTrace[],
+ limits: TraceBatchWeightLimits,
+) =>
+ batch.length <= 1 ||
+ fitsWeightLimits(
+ batch.reduce((total, entry) => total + getEventUpdates(entry), 0),
+ batch.reduce((total, entry) => total + getSerializedEventBytes(entry), 0),
+ limits,
+ );
+
+// A heavy trace uses more than half of a batch budget.
+const isHeavyTrace = (entry: PendingTrace, limits: TraceBatchWeightLimits) =>
+ getEventUpdates(entry) * 2 > limits.maxEventUpdates ||
+ getSerializedEventBytes(entry) * 2 > limits.maxSerializedEventBytes;
+
+type TraceExclusionReason = "event_updates" | "serialized_bytes";
+
+const getTraceExclusionReason = (
+ { estimates }: PendingTrace,
+ limits: TraceBatchWeightLimits,
+): TraceExclusionReason | undefined => {
+ if (!estimates) return undefined;
+ if (estimates.eventUpdateCount > limits.maxEventUpdates)
+ return "event_updates";
+ if (estimates.serializedEventBytes > limits.maxSerializedEventBytes)
+ return "serialized_bytes";
+ return undefined;
+};
+
const compareNumbers = (left: number, right: number) => left - right;
const compareStrings = (left: string, right: string) => {
@@ -122,8 +184,10 @@ const getBatchBounds = (batch: readonly PendingTrace[]) => ({
const isFeasibleLocalityBatch = (
batch: readonly PendingTrace[],
maxBatchSize: number,
+ weightLimits: TraceBatchWeightLimits,
) => {
if (batch.length === 0 || batch.length > maxBatchSize) return false;
+ if (!isWithinWeightLimits(batch, weightLimits)) return false;
const canonical = canonicalizeLocalityBatch(batch);
const { minStart, maxStart } = getBatchBounds(canonical);
return maxStart - minStart <= allowedEnvelopeMs(canonical[0]);
@@ -156,6 +220,7 @@ const ZERO_BATCH_SELECTION_COST: BatchSelectionCost = {
const selectLocalityBatches = (
candidates: readonly PendingTrace[],
maxBatchSize: number,
+ weightLimits: TraceBatchWeightLimits,
): PendingTrace[][] => {
const sorted = candidates
.map(toLocalityCandidate)
@@ -189,10 +254,20 @@ const selectLocalityBatches = (
let maxMinute = Number.NEGATIVE_INFINITY;
let minStart = Number.POSITIVE_INFINITY;
let maxStart = Number.NEGATIVE_INFINITY;
+ let eventUpdates = 0;
+ let serializedEventBytes = 0;
const seedEnvelopeMs = allowedEnvelopeMs(sorted[start].entry);
const maxLength = Math.min(maxBatchSize, candidateCount - start);
for (let length = 1; length <= maxLength; length++) {
const candidate = sorted[start + length - 1];
+ eventUpdates += getEventUpdates(candidate.entry);
+ serializedEventBytes += getSerializedEventBytes(candidate.entry);
+ // Weights are nonnegative, so every longer slice is also over budget.
+ if (
+ length > 1 &&
+ !fitsWeightLimits(eventUpdates, serializedEventBytes, weightLimits)
+ )
+ break;
minMinute = Math.min(minMinute, candidate.minMinute);
maxMinute = Math.max(maxMinute, candidate.maxMinute);
minStart = Math.min(minStart, candidate.entry.trace.minStart);
@@ -298,6 +373,9 @@ const selectLocalityBatches = (
* that envelope cap forbids a cheaper fill. Among feasible partitions with
* that minimum job count, dynamic programming first avoids crossing projects,
* then minimizes minute × trace-hash cells, then splits the largest hash gaps.
+ * Both strategies also close a multi-trace batch before its estimated event
+ * updates or serialized bytes would exceed `weightLimits`. Locality packs
+ * traces above half of either budget in a separate lane.
*
* Selection uses O(n × k × maxBatchSize) time and
* O(n × min(n, maxBatchSize) + n × k) memory, where k is the fewest feasible
@@ -310,17 +388,52 @@ export function selectTraceBatches(
candidates: readonly PendingTrace[],
maxBatchSize: number,
strategy: TraceBatchStrategy,
+ weightLimits: TraceBatchWeightLimits = UNBOUNDED_WEIGHT_LIMITS,
): PendingTrace[][] {
if (candidates.length === 0) return [];
if (strategy === "project") {
const batches: PendingTrace[][] = [];
- for (let offset = 0; offset < candidates.length; offset += maxBatchSize) {
- batches.push(candidates.slice(offset, offset + maxBatchSize));
+ let batch: PendingTrace[] = [];
+ let eventUpdates = 0;
+ let serializedEventBytes = 0;
+ for (const candidate of candidates) {
+ const candidateEventUpdates = getEventUpdates(candidate);
+ const candidateBytes = getSerializedEventBytes(candidate);
+ if (
+ batch.length > 0 &&
+ (batch.length === maxBatchSize ||
+ !fitsWeightLimits(
+ eventUpdates + candidateEventUpdates,
+ serializedEventBytes + candidateBytes,
+ weightLimits,
+ ))
+ ) {
+ batches.push(batch);
+ batch = [];
+ eventUpdates = 0;
+ serializedEventBytes = 0;
+ }
+ batch.push(candidate);
+ eventUpdates += candidateEventUpdates;
+ serializedEventBytes += candidateBytes;
}
+ batches.push(batch);
return batches;
}
- return selectLocalityBatches(candidates, maxBatchSize);
+ // Two heavy traces cannot share a batch on the dimension that makes them
+ // heavy, so packing them separately keeps them from fragmenting the light
+ // traces around them in locality order.
+ const light: PendingTrace[] = [];
+ const heavy: PendingTrace[] = [];
+ for (const candidate of candidates) {
+ if (isHeavyTrace(candidate, weightLimits)) heavy.push(candidate);
+ else light.push(candidate);
+ }
+ return [
+ ...selectLocalityBatches(light, maxBatchSize, weightLimits),
+ ...selectLocalityBatches(heavy, maxBatchSize, weightLimits),
+ ];
}
const compareBatchesByOldestDue = (
@@ -336,6 +449,7 @@ const canCoalesceLocalityPartials = (
left: readonly PendingTrace[],
right: readonly PendingTrace[],
maxBatchSize: number,
+ weightLimits: TraceBatchWeightLimits,
) => {
if (left.length + right.length > maxBatchSize) return false;
const projectId = left[0]?.trace.projectId;
@@ -355,7 +469,7 @@ const canCoalesceLocalityPartials = (
leftBounds.maxStart + QUERY_BUFFER_MS;
return (
bufferedIntervalsOverlap &&
- isFeasibleLocalityBatch([...left, ...right], maxBatchSize)
+ isFeasibleLocalityBatch([...left, ...right], maxBatchSize, weightLimits)
);
};
@@ -367,6 +481,7 @@ const canCoalesceLocalityPartials = (
export function prepareLocalityPartials(
partials: readonly (readonly PendingTrace[])[],
maxBatchSize: number,
+ weightLimits: TraceBatchWeightLimits = UNBOUNDED_WEIGHT_LIMITS,
): { dispatch: PendingTrace[][]; carry: PendingTrace[][] } {
const batches = partials
.filter((batch) => batch.length > 0)
@@ -390,7 +505,10 @@ export function prepareLocalityPartials(
) {
const left = batches[leftIndex];
const right = batches[rightIndex];
- if (!canCoalesceLocalityPartials(left, right, maxBatchSize)) continue;
+ if (
+ !canCoalesceLocalityPartials(left, right, maxBatchSize, weightLimits)
+ )
+ continue;
const leftBounds = getBatchBounds(left);
const rightBounds = getBatchBounds(right);
const merged = canonicalizeLocalityBatch([...left, ...right]);
@@ -535,14 +653,18 @@ const HYDRATE_SCRIPT = `
return result
`;
+// Returns one flag per member, in argument order: 1 when removed, 0 when a
+// newer revision keeps the trace pending.
const ACKNOWLEDGE_SCRIPT = `
- local removed = 0
+ local removed = {}
for i = 1, #ARGV, 2 do
local state = redis.call('HGET', KEYS[2], ARGV[i])
if state and cjson.decode(state).revision == ARGV[i + 1] then
redis.call('ZREM', KEYS[1], ARGV[i])
redis.call('HDEL', KEYS[2], ARGV[i])
- removed = removed + 1
+ table.insert(removed, 1)
+ else
+ table.insert(removed, 0)
end
end
return removed
@@ -789,6 +911,66 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
return 0;
});
recordDistribution("langfuse.trace_batch.snapshot_size", members.length);
+ const batchWeightLimits: TraceBatchWeightLimits = {
+ maxEventUpdates: toLimit(
+ env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES,
+ ),
+ maxSerializedEventBytes: toLimit(
+ env.LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES,
+ ),
+ };
+ const traceWeightLimits: TraceBatchWeightLimits = {
+ maxEventUpdates: toLimit(
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES,
+ ),
+ maxSerializedEventBytes: toLimit(
+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES,
+ ),
+ };
+
+ const acknowledge = async (
+ batch: readonly PendingTrace[],
+ ): Promise<boolean[]> =>
+ (
+ (await redis!.eval(
+ ACKNOWLEDGE_SCRIPT,
+ 2,
+ DUE_KEY,
+ STATE_KEY,
+ ...batch.flatMap(({ member, trace }) => [member, trace.revision]),
+ )) as number[]
+ ).map((flag) => Number(flag) === 1);
+
+ // Excluded traces leave Redis like dispatched ones; activity after this
+ // run's snapshot starts a fresh pending window that is evaluated again.
+ // A trace whose revision changed before acknowledgement stays pending and
+ // is reported only once a later run removes it.
+ const exclude = async (
+ excluded: readonly {
+ entry: PendingTrace;
+ reason: TraceExclusionReason;
+ }[],
+ ) => {
+ if (excluded.length === 0) return;
+ await this.extendLockOnProgress(true);
+ if (this.stopping) return;
+ const removed = await acknowledge(excluded.map(({ entry }) => entry));
+ for (const [index, { entry, reason }] of excluded.entries()) {
+ if (!removed[index]) continue;
+ recordIncrement("langfuse.trace_batch.excluded_traces", 1, {
+ reason,
+ strategy,
+ });
+ logger.warn("Trace batch excluded oversized trace", {
+ projectId: entry.trace.projectId,
+ traceId: entry.trace.traceId,
+ reason,
+ estimatedEventUpdateCount: entry.estimates?.eventUpdateCount,
+ estimatedSerializedEventBytes: entry.estimates?.serializedEventBytes,
+ eventTimeEnvelopeMs: entry.trace.maxStart - entry.trace.minStart,
+ });
+ }
+ };
const enqueue = async (batch: PendingTrace[]) => {
await this.extendLockOnProgress(true);
@@ -880,18 +1062,10 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
strategy,
});
}
- const removed = Number(
- await redis!.eval(
- ACKNOWLEDGE_SCRIPT,
- 2,
- DUE_KEY,
- STATE_KEY,
- ...batch.flatMap(({ member, trace }) => [member, trace.revision]),
- ),
- );
+ const removed = await acknowledge(batch);
recordIncrement(
"langfuse.trace_batch.reactivated_traces",
- batch.length - removed,
+ removed.filter((wasRemoved) => !wasRemoved).length,
);
};
@@ -918,12 +1092,16 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
candidates.length - hydrated.length / 3,
);
const hydratedCandidates: PendingTrace[] = [];
+ const excluded: {
+ entry: PendingTrace;
+ reason: TraceExclusionReason;
+ }[] = [];
for (let i = 0; i < hydrated.length; i += 3) {
if (this.stopping) return;
const member = hydrated[i];
const [projectId, traceId] = JSON.parse(member) as [string, string];
const state = JSON.parse(hydrated[i + 1]);
- hydratedCandidates.push({
+ const entry: PendingTrace = {
member,
due: Number(hydrated[i + 2]),
estimates:
@@ -941,8 +1119,13 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
projectId,
traceId,
}),
- });
+ };
+ const reason = getTraceExclusionReason(entry, traceWeightLimits);
+ if (reason) excluded.push({ entry, reason });
+ else hydratedCandidates.push(entry);
}
+ await exclude(excluded);
+ if (this.stopping) return;
const carriedTraceCount =
strategy === "project"
@@ -962,6 +1145,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
selectorInput,
env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
strategy,
+ batchWeightLimits,
);
recordDistribution(
"langfuse.trace_batch.selector_duration_ms",
@@ -996,6 +1180,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
const prepared = prepareLocalityPartials(
partialBatches,
env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
+ batchWeightLimits,
);
localityPartials = prepared.carry;
for (const batch of prepared.dispatch) {