🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(trace-batch): bound batches by estimated weight and exclude oversized traces (#18296)
Lysander Kieselc49954f1caf8
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex 4fe9f45858..49841ff91d 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -410,6 +410,111 @@ describe("trace batch selection", () => {expect(all.every((batch) => batch.length <= 5)).toBe(true);expect(new Set(all.flat().map(({ member }) => member)).size).toBe(10);});++ describe("weight limits", () => {+ const weighted = (+ traceId: string,+ eventUpdateCount: number,+ serializedEventBytes = eventUpdateCount * 1_000,+ ): PendingTrace => ({+ ...pendingTrace("project", traceId, 1, 0, minute),+ estimates: { eventUpdateCount, serializedEventBytes },+ });+ const limits = { maxEventUpdates: 10, maxSerializedEventBytes: 50_000 };+ const weightOf = (batch: PendingTrace[]) =>+ batch.reduce(+ (total, { estimates }) => ({+ events: total.events + (estimates?.eventUpdateCount ?? 0),+ bytes: total.bytes + (estimates?.serializedEventBytes ?? 0),+ }),+ { events: 0, bytes: 0 },+ );++ it.each(["project", "locality"] as const)(+ "closes %s batches before the event or byte budget and isolates overweight traces",+ (strategy) => {+ const candidates = [+ weighted("a", 4),+ weighted("b", 4),+ weighted("c", 4),+ weighted("heavy", 25),+ weighted("bytes-1", 1, 30_000),+ weighted("bytes-2", 1, 30_000),+ { ...pendingTrace("project", "unknown", 1, 0, minute) },+ ];++ const batches = selectTraceBatches(candidates, 60, strategy, limits);++ expect(batches.flat()).toHaveLength(candidates.length);+ for (const batch of batches) {+ if (batch.length === 1) continue;+ const { events, bytes } = weightOf(batch);+ expect(events).toBeLessThanOrEqual(limits.maxEventUpdates);+ expect(bytes).toBeLessThanOrEqual(limits.maxSerializedEventBytes);+ }+ expect(+ batches.find((batch) =>+ batch.some(({ trace }) => trace.traceId === "heavy"),+ ),+ ).toHaveLength(1);+ expect(+ batches.some(+ (batch) =>+ batch.some(({ trace }) => trace.traceId === "bytes-1") &&+ batch.some(({ trace }) => trace.traceId === "bytes-2"),+ ),+ ).toBe(false);+ // Without budgets the same candidates fit one batch.+ expect(selectTraceBatches(candidates, 60, strategy)).toHaveLength(1);+ },+ );++ it("packs light locality neighbors of a heavy trace into full batches", () => {+ const at = (+ traceId: string,+ start: number,+ eventUpdateCount: number,+ ): PendingTrace => ({+ ...pendingTrace("project", traceId, 1, start, start),+ estimates: {+ eventUpdateCount,+ serializedEventBytes: eventUpdateCount * 1_000,+ },+ });+ // Locality order: two light traces, the heavy trace, six light traces.+ const candidates = [+ at("light-0", 0, 1),+ at("light-1", 0, 1),+ at("heavy", minute, 9),+ ...Array.from({ length: 6 }, (_, index) =>+ at(`light-${index + 2}`, 2 * minute, 1),+ ),+ ];++ const batches = selectTraceBatches(candidates, 4, "locality", limits);++ expect(batches.flat()).toHaveLength(candidates.length);+ expect(batches).toHaveLength(3);+ expect(+ batches.find((batch) =>+ batch.some(({ trace }) => trace.traceId === "heavy"),+ ),+ ).toHaveLength(1);+ });++ it("does not coalesce locality partials past the weight budget", () => {+ const result = prepareLocalityPartials(+ [[weighted("a", 6)], [weighted("b", 6)], [weighted("c", 4)]],+ 60,+ limits,+ );+ const all = [...result.dispatch, ...result.carry];++ expect(all.flat()).toHaveLength(3);+ expect(all.every((batch) => weightOf(batch).events <= 10)).toBe(true);+ expect(all).toHaveLength(2);+ });+ });});describe("trace micro-batch scheduling with Redis", () => {@@ -426,6 +531,10 @@ describe("trace micro-batch scheduling with Redis", () => {const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;+ const originalMaxTraceEventUpdates =+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES;+ const originalMaxTraceBytes =+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES;let queue: Queue<TQueueJobTypes[QueueName.TraceBatch]>;let connection: NonNullable<ReturnType<typeof createNewRedisInstance>>;const runners: TraceBatchDispatcher[] = [];@@ -493,6 +602,9 @@ describe("trace micro-batch scheduling with Redis", () => {env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES =+ originalMaxTraceEventUpdates;+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES = originalMaxTraceBytes;await queue.obliterate({ force: true });await queue.close();connection.disconnect();@@ -662,6 +774,93 @@ describe("trace micro-batch scheduling with Redis", () => {).toBe(true);});+ it("excludes oversized traces from dispatch, removes their state, and reports their IDs", async () => {+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES = 3;+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES = 1_000;+ const warn = vi.spyOn(shared.logger, "warn");+ await trackTraceBatchActivity("project", [+ ...Array.from({ length: 4 }, (_, index) =>+ event("many-updates", 1_000_000 + index),+ ),+ event("large-input", 1_000_000, 5_000),+ event("small", 1_000_000),+ ]);+ await makeDue("project", "many-updates", "large-input", "small");++ await runner().processBatch();++ const jobs = await queue.getJobs(["wait"]);+ expect(+ jobs.map((job) => job.data.payload.traces.map(({ traceId }) => traceId)),+ ).toEqual([["small"]]);+ expect(await client().zcard(dueKey)).toBe(0);+ expect(await client().hlen(stateKey)).toBe(0);+ for (const [traceId, reason] of [+ ["many-updates", "event_updates"],+ ["large-input", "serialized_bytes"],+ ] as const) {+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.excluded_traces",+ 1,+ { reason, strategy: "project" },+ );+ expect(warn).toHaveBeenCalledWith(+ "Trace batch excluded oversized trace",+ expect.objectContaining({ projectId: "project", traceId, reason }),+ );+ }+ });++ it("does not report an oversized trace as excluded when new activity keeps it pending", async () => {+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES = 3;+ const warn = vi.spyOn(shared.logger, "warn");+ await trackTraceBatchActivity(+ "project",+ Array.from({ length: 4 }, (_, index) =>+ event("reactivated", 1_000_000 + index),+ ),+ );+ await makeDue("project", "reactivated");+ const evaluate = client().eval.bind(client());+ let reactivated = false;+ vi.spyOn(client(), "eval").mockImplementation((async (+ script: string,+ ...args: (string | number)[]+ ) => {+ if (!reactivated && script.includes("cjson.decode(state).revision ==")) {+ reactivated = true;+ const traceMember = member("project", "reactivated");+ const state = JSON.parse((await client().hget(stateKey, traceMember))!);+ await client().hset(+ stateKey,+ traceMember,+ JSON.stringify({ ...state, revision: randomUUID() }),+ );+ }+ return evaluate(script, ...args);+ }) as typeof evaluate);++ await runner().processBatch();++ expect(reactivated).toBe(true);+ expect(+ await client().hexists(stateKey, member("project", "reactivated")),+ ).toBe(1);+ expect(recordIncrement).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.excluded_traces",+ expect.anything(),+ expect.anything(),+ );+ expect(warn).not.toHaveBeenCalledWith(+ "Trace batch excluded oversized trace",+ expect.anything(),+ );+ expect(recordIncrement).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.reactivated_traces",+ 1,+ );+ });+it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;const evaluate = vi.spyOn(client(), "eval");diff --git a/worker/src/env.ts b/worker/src/env.tsindex 288f17a32a..f7a7cce803 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -151,6 +151,27 @@ const EnvSchema = z.object({.positive().max(10_000).default(60),+ // Weight budgets use ingestion-time estimates; 0 disables a budget.+ LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES: z.coerce+ .number()+ .int()+ .nonnegative()+ .default(1_500),+ LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES: z.coerce+ .number()+ .int()+ .nonnegative()+ .default(64 * 1024 * 1024),+ LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES: z.coerce+ .number()+ .int()+ .nonnegative()+ .default(5_000),+ LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES: z.coerce+ .number()+ .int()+ .nonnegative()+ .default(50 * 1024 * 1024),LANGFUSE_TRACE_BATCH_STRATEGY: z.enum(["project", "locality"]).default("project"),diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 2bc8324c31..94d9dd97da 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -9,23 +9,27 @@ absent from env templates.## Controls-| Variable | Default | Role |-| ------------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------- |-| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |-| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |-| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |-| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |-| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |-| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |-| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |-| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |-| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |-| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |-| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |-| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |-| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |-| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |+| Variable | Default | Role |+| ------------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------- |+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |+| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |+| `LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES` | `1500` | Estimated event updates per multi-trace job; `0` disables. |+| `LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES` | `67108864` | Estimated serialized event bytes per multi-trace job (64 MiB); `0` disables. |+| `LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` | `5000` | Exclude a trace whose pending window exceeds this many event updates; `0` disables. |+| `LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` | `52428800` | Exclude a trace whose pending window exceeds these serialized bytes (50 MiB); `0` disables. |+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |Producer, dispatcher and consumer can run independently. Sampling reuses theevaluator's deterministic trace-ID sampler; every observation of the same trace@@ -57,7 +61,7 @@ than retention; ingestion and dispatcher both perform cleanup.One dispatcher holds the renewable lease. It snapshots ready IDs, sorts them byproject, then hydrates and revalidates state in chunks of 1,000. Jobs contain up-to the configured trace cap; project leftovers can share a job. The dispatcher+to the configured trace cap and weight budgets; project leftovers can share a job. The dispatcherenqueues before acknowledging matching revisions, so concurrent intake stayspending and failed enqueueing can be retried. Stable job IDs limit duplicates.@@ -76,17 +80,39 @@ The dispatcher emits `estimated_event_update_count` and`estimated_serialized_event_bytes` distributions under `langfuse.trace_batch`, taggedwith `scope:trace|batch` and `strategy`. Batch totals require known estimates forevery member; `estimates_unavailable` counts unknown traces and affected batches.-Estimates stay out of queue payloads, job IDs and batch selection. They do not cap-work or change oversized-trace handling. Dispatch retries can emit another sample.+Estimates stay out of queue payloads and job IDs. Dispatch retries can emit+another sample.Serialized event bytes differ from the reader's `input_bytes`, `output_bytes`and `metadata_bytes` metrics: they include the serialized event fields and JSONencoding, and do not represent RAM, network transfer or ClickHouse scan bytes.The job return still sums those three as `ioMetadataBytes`; that sum is not a-custom metric. After rollout, collect a stable hour-with unchanged sampling and batching settings; compare trace/batch distributions,-unknown-estimate counts, ingestion latency and Redis command rate/CPU/memory with-the baseline before choosing any size-aware batching policy.+custom metric.++### Weight budgets and oversized traces++Both strategies close a job before its summed estimates would exceed+`LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES` or+`LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES`, in addition to the trace cap.+Locality treats an over-budget slice like one past its time envelope, and+partials coalesce only within the budget. A single trace always forms a job, so+a trace heavier than the budget runs alone. Locality packs traces above half of+either budget in a separate lane, so one heavy trace does not split the light+traces around it into extra jobs. Unknown estimates weigh nothing.+Budget-limited jobs report `fill:partial`; the `scope:batch` estimate+distributions show how close jobs run to the budget.++After hydration, a trace whose pending-window estimates exceed+`LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES` or+`LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES` is never queued or read. The+dispatcher acknowledges it like a dispatched trace, increments+`langfuse.trace_batch.excluded_traces` tagged with+`reason:event_updates|serialized_bytes` and `strategy`, and logs+`Trace batch excluded oversized trace` at warn level with `projectId`, `traceId`,+`reason` and both estimates. Group that log by `@projectId` to find repeat+offenders. Activity after the acknowledgement starts a new pending window that+is evaluated again, so a trace that keeps growing may be logged more than once.+Traces with unknown estimates are never excluded.The worker streams full `events_full` payloads and records batch observation,trace and logical payload-byte metrics. The `input_bytes`, `output_bytes`@@ -105,8 +131,9 @@ This fixed margin neither waits for arrivals nor guarantees trace completeness;repeated intervals for the same pair form a union without duplicating rows.Queries use response compression and chunked multipart parameters for up to10,000 UUID-sized trace/project IDs. The fixed 30-second timeout fails the job-rather than accepting partial results. This does not cap observation count,-query memory or dispatcher snapshot memory.+rather than accepting partial results. Beyond the dispatcher's weight budgets+and trace exclusion, this does not cap observation count, query memory or+dispatcher snapshot memory.## Per-trace transcripts@@ -457,8 +484,8 @@ project ordering and carries unfinished jobs across hydration chunks.Locality sorts each hydrated window by project, minimum start minute, maximumstart minute and `xxHash32(traceId)`. It finds the fewest feasible jobs under the-trace cap and an event-time envelope of at most one hour, or 125% of the first-trace's observed span if larger. Within that job count, it minimizes project+trace cap, the weight budgets and an event-time envelope of at most one hour, or+125% of the first trace's observed span if larger. Within that job count, it minimizes projectboundaries, then minute × trace count, then gaps between trace hashes. These areread-locality proxies, not measured ClickHouse scan costs.diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex 73f1f81e6b..5bb400ca01 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -37,6 +37,68 @@ export type PendingTrace = {trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];};+/**+ * Upper bounds on the summed ingestion estimates of one dispatched batch.+ * Traces without estimates weigh nothing, and a single trace always forms a+ * batch even when it alone exceeds a budget.+ */+export type TraceBatchWeightLimits = {+ maxEventUpdates: number;+ maxSerializedEventBytes: number;+};++const UNBOUNDED_WEIGHT_LIMITS: TraceBatchWeightLimits = {+ maxEventUpdates: Number.POSITIVE_INFINITY,+ maxSerializedEventBytes: Number.POSITIVE_INFINITY,+};++const toLimit = (value: number) =>+ value > 0 ? value : Number.POSITIVE_INFINITY;++const getEventUpdates = ({ estimates }: PendingTrace) =>+ estimates?.eventUpdateCount ?? 0;++const getSerializedEventBytes = ({ estimates }: PendingTrace) =>+ estimates?.serializedEventBytes ?? 0;++const fitsWeightLimits = (+ eventUpdates: number,+ serializedEventBytes: number,+ limits: TraceBatchWeightLimits,+) =>+ eventUpdates <= limits.maxEventUpdates &&+ serializedEventBytes <= limits.maxSerializedEventBytes;++const isWithinWeightLimits = (+ batch: readonly PendingTrace[],+ limits: TraceBatchWeightLimits,+) =>+ batch.length <= 1 ||+ fitsWeightLimits(+ batch.reduce((total, entry) => total + getEventUpdates(entry), 0),+ batch.reduce((total, entry) => total + getSerializedEventBytes(entry), 0),+ limits,+ );++// A heavy trace uses more than half of a batch budget.+const isHeavyTrace = (entry: PendingTrace, limits: TraceBatchWeightLimits) =>+ getEventUpdates(entry) * 2 > limits.maxEventUpdates ||+ getSerializedEventBytes(entry) * 2 > limits.maxSerializedEventBytes;++type TraceExclusionReason = "event_updates" | "serialized_bytes";++const getTraceExclusionReason = (+ { estimates }: PendingTrace,+ limits: TraceBatchWeightLimits,+): TraceExclusionReason | undefined => {+ if (!estimates) return undefined;+ if (estimates.eventUpdateCount > limits.maxEventUpdates)+ return "event_updates";+ if (estimates.serializedEventBytes > limits.maxSerializedEventBytes)+ return "serialized_bytes";+ return undefined;+};+const compareNumbers = (left: number, right: number) => left - right;const compareStrings = (left: string, right: string) => {@@ -122,8 +184,10 @@ const getBatchBounds = (batch: readonly PendingTrace[]) => ({const isFeasibleLocalityBatch = (batch: readonly PendingTrace[],maxBatchSize: number,+ weightLimits: TraceBatchWeightLimits,) => {if (batch.length === 0 || batch.length > maxBatchSize) return false;+ if (!isWithinWeightLimits(batch, weightLimits)) return false;const canonical = canonicalizeLocalityBatch(batch);const { minStart, maxStart } = getBatchBounds(canonical);return maxStart - minStart <= allowedEnvelopeMs(canonical[0]);@@ -156,6 +220,7 @@ const ZERO_BATCH_SELECTION_COST: BatchSelectionCost = {const selectLocalityBatches = (candidates: readonly PendingTrace[],maxBatchSize: number,+ weightLimits: TraceBatchWeightLimits,): PendingTrace[][] => {const sorted = candidates.map(toLocalityCandidate)@@ -189,10 +254,20 @@ const selectLocalityBatches = (let maxMinute = Number.NEGATIVE_INFINITY;let minStart = Number.POSITIVE_INFINITY;let maxStart = Number.NEGATIVE_INFINITY;+ let eventUpdates = 0;+ let serializedEventBytes = 0;const seedEnvelopeMs = allowedEnvelopeMs(sorted[start].entry);const maxLength = Math.min(maxBatchSize, candidateCount - start);for (let length = 1; length <= maxLength; length++) {const candidate = sorted[start + length - 1];+ eventUpdates += getEventUpdates(candidate.entry);+ serializedEventBytes += getSerializedEventBytes(candidate.entry);+ // Weights are nonnegative, so every longer slice is also over budget.+ if (+ length > 1 &&+ !fitsWeightLimits(eventUpdates, serializedEventBytes, weightLimits)+ )+ break;minMinute = Math.min(minMinute, candidate.minMinute);maxMinute = Math.max(maxMinute, candidate.maxMinute);minStart = Math.min(minStart, candidate.entry.trace.minStart);@@ -298,6 +373,9 @@ const selectLocalityBatches = (* that envelope cap forbids a cheaper fill. Among feasible partitions with* that minimum job count, dynamic programming first avoids crossing projects,* then minimizes minute × trace-hash cells, then splits the largest hash gaps.+ * Both strategies also close a multi-trace batch before its estimated event+ * updates or serialized bytes would exceed `weightLimits`. Locality packs+ * traces above half of either budget in a separate lane.** Selection uses O(n × k × maxBatchSize) time and* O(n × min(n, maxBatchSize) + n × k) memory, where k is the fewest feasible@@ -310,17 +388,52 @@ export function selectTraceBatches(candidates: readonly PendingTrace[],maxBatchSize: number,strategy: TraceBatchStrategy,+ weightLimits: TraceBatchWeightLimits = UNBOUNDED_WEIGHT_LIMITS,): PendingTrace[][] {if (candidates.length === 0) return [];if (strategy === "project") {const batches: PendingTrace[][] = [];- for (let offset = 0; offset < candidates.length; offset += maxBatchSize) {- batches.push(candidates.slice(offset, offset + maxBatchSize));+ let batch: PendingTrace[] = [];+ let eventUpdates = 0;+ let serializedEventBytes = 0;+ for (const candidate of candidates) {+ const candidateEventUpdates = getEventUpdates(candidate);+ const candidateBytes = getSerializedEventBytes(candidate);+ if (+ batch.length > 0 &&+ (batch.length === maxBatchSize ||+ !fitsWeightLimits(+ eventUpdates + candidateEventUpdates,+ serializedEventBytes + candidateBytes,+ weightLimits,+ ))+ ) {+ batches.push(batch);+ batch = [];+ eventUpdates = 0;+ serializedEventBytes = 0;+ }+ batch.push(candidate);+ eventUpdates += candidateEventUpdates;+ serializedEventBytes += candidateBytes;}+ batches.push(batch);return batches;}- return selectLocalityBatches(candidates, maxBatchSize);+ // Two heavy traces cannot share a batch on the dimension that makes them+ // heavy, so packing them separately keeps them from fragmenting the light+ // traces around them in locality order.+ const light: PendingTrace[] = [];+ const heavy: PendingTrace[] = [];+ for (const candidate of candidates) {+ if (isHeavyTrace(candidate, weightLimits)) heavy.push(candidate);+ else light.push(candidate);+ }+ return [+ ...selectLocalityBatches(light, maxBatchSize, weightLimits),+ ...selectLocalityBatches(heavy, maxBatchSize, weightLimits),+ ];}const compareBatchesByOldestDue = (@@ -336,6 +449,7 @@ const canCoalesceLocalityPartials = (left: readonly PendingTrace[],right: readonly PendingTrace[],maxBatchSize: number,+ weightLimits: TraceBatchWeightLimits,) => {if (left.length + right.length > maxBatchSize) return false;const projectId = left[0]?.trace.projectId;@@ -355,7 +469,7 @@ const canCoalesceLocalityPartials = (leftBounds.maxStart + QUERY_BUFFER_MS;return (bufferedIntervalsOverlap &&- isFeasibleLocalityBatch([...left, ...right], maxBatchSize)+ isFeasibleLocalityBatch([...left, ...right], maxBatchSize, weightLimits));};@@ -367,6 +481,7 @@ const canCoalesceLocalityPartials = (export function prepareLocalityPartials(partials: readonly (readonly PendingTrace[])[],maxBatchSize: number,+ weightLimits: TraceBatchWeightLimits = UNBOUNDED_WEIGHT_LIMITS,): { dispatch: PendingTrace[][]; carry: PendingTrace[][] } {const batches = partials.filter((batch) => batch.length > 0)@@ -390,7 +505,10 @@ export function prepareLocalityPartials() {const left = batches[leftIndex];const right = batches[rightIndex];- if (!canCoalesceLocalityPartials(left, right, maxBatchSize)) continue;+ if (+ !canCoalesceLocalityPartials(left, right, maxBatchSize, weightLimits)+ )+ continue;const leftBounds = getBatchBounds(left);const rightBounds = getBatchBounds(right);const merged = canonicalizeLocalityBatch([...left, ...right]);@@ -535,14 +653,18 @@ const HYDRATE_SCRIPT = `return result`;+// Returns one flag per member, in argument order: 1 when removed, 0 when a+// newer revision keeps the trace pending.const ACKNOWLEDGE_SCRIPT = `- local removed = 0+ local removed = {}for i = 1, #ARGV, 2 dolocal state = redis.call('HGET', KEYS[2], ARGV[i])if state and cjson.decode(state).revision == ARGV[i + 1] thenredis.call('ZREM', KEYS[1], ARGV[i])redis.call('HDEL', KEYS[2], ARGV[i])- removed = removed + 1+ table.insert(removed, 1)+ else+ table.insert(removed, 0)endendreturn removed@@ -789,6 +911,66 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {return 0;});recordDistribution("langfuse.trace_batch.snapshot_size", members.length);+ const batchWeightLimits: TraceBatchWeightLimits = {+ maxEventUpdates: toLimit(+ env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES,+ ),+ maxSerializedEventBytes: toLimit(+ env.LANGFUSE_TRACE_BATCH_MAX_BATCH_SERIALIZED_BYTES,+ ),+ };+ const traceWeightLimits: TraceBatchWeightLimits = {+ maxEventUpdates: toLimit(+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES,+ ),+ maxSerializedEventBytes: toLimit(+ env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES,+ ),+ };++ const acknowledge = async (+ batch: readonly PendingTrace[],+ ): Promise<boolean[]> =>+ (+ (await redis!.eval(+ ACKNOWLEDGE_SCRIPT,+ 2,+ DUE_KEY,+ STATE_KEY,+ ...batch.flatMap(({ member, trace }) => [member, trace.revision]),+ )) as number[]+ ).map((flag) => Number(flag) === 1);++ // Excluded traces leave Redis like dispatched ones; activity after this+ // run's snapshot starts a fresh pending window that is evaluated again.+ // A trace whose revision changed before acknowledgement stays pending and+ // is reported only once a later run removes it.+ const exclude = async (+ excluded: readonly {+ entry: PendingTrace;+ reason: TraceExclusionReason;+ }[],+ ) => {+ if (excluded.length === 0) return;+ await this.extendLockOnProgress(true);+ if (this.stopping) return;+ const removed = await acknowledge(excluded.map(({ entry }) => entry));+ for (const [index, { entry, reason }] of excluded.entries()) {+ if (!removed[index]) continue;+ recordIncrement("langfuse.trace_batch.excluded_traces", 1, {+ reason,+ strategy,+ });+ logger.warn("Trace batch excluded oversized trace", {+ projectId: entry.trace.projectId,+ traceId: entry.trace.traceId,+ reason,+ estimatedEventUpdateCount: entry.estimates?.eventUpdateCount,+ estimatedSerializedEventBytes: entry.estimates?.serializedEventBytes,+ eventTimeEnvelopeMs: entry.trace.maxStart - entry.trace.minStart,+ });+ }+ };const enqueue = async (batch: PendingTrace[]) => {await this.extendLockOnProgress(true);@@ -880,18 +1062,10 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {strategy,});}- const removed = Number(- await redis!.eval(- ACKNOWLEDGE_SCRIPT,- 2,- DUE_KEY,- STATE_KEY,- ...batch.flatMap(({ member, trace }) => [member, trace.revision]),- ),- );+ const removed = await acknowledge(batch);recordIncrement("langfuse.trace_batch.reactivated_traces",- batch.length - removed,+ removed.filter((wasRemoved) => !wasRemoved).length,);};@@ -918,12 +1092,16 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {candidates.length - hydrated.length / 3,);const hydratedCandidates: PendingTrace[] = [];+ const excluded: {+ entry: PendingTrace;+ reason: TraceExclusionReason;+ }[] = [];for (let i = 0; i < hydrated.length; i += 3) {if (this.stopping) return;const member = hydrated[i];const [projectId, traceId] = JSON.parse(member) as [string, string];const state = JSON.parse(hydrated[i + 1]);- hydratedCandidates.push({+ const entry: PendingTrace = {member,due: Number(hydrated[i + 2]),estimates:@@ -941,8 +1119,13 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {projectId,traceId,}),- });+ };+ const reason = getTraceExclusionReason(entry, traceWeightLimits);+ if (reason) excluded.push({ entry, reason });+ else hydratedCandidates.push(entry);}+ await exclude(excluded);+ if (this.stopping) return;const carriedTraceCount =strategy === "project"@@ -962,6 +1145,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {selectorInput,env.LANGFUSE_TRACE_BATCH_MAX_SIZE,strategy,+ batchWeightLimits,);recordDistribution("langfuse.trace_batch.selector_duration_ms",@@ -996,6 +1180,7 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {const prepared = prepareLocalityPartials(partialBatches,env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ batchWeightLimits,);localityPartials = prepared.carry;for (const batch of prepared.dispatch) {