🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(trace-batching): measure pending trace size estimates (#17586)
Lysander Kiesel536e38e5bcdc
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex 7c4f5a71df..6536c90455 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -430,9 +430,14 @@ describe("trace micro-batch scheduling with Redis", () => {};const member = (projectId: string, traceId: string) =>JSON.stringify([projectId, traceId]);- const event = (traceId: string, start = 1_000_000) => ({+ const event = (+ traceId: string,+ start = 1_000_000,+ serializedEventBytes = 100,+ ) => ({traceId,startTimeISO: new Date(start).toISOString(),+ serializedEventBytes,});const runner = () => {const instance = new TraceBatchDispatcher();@@ -822,24 +827,61 @@ describe("trace micro-batch scheduling with Redis", () => {it("preserves min/max across concurrent out-of-order arrivals, resets readiness, and separates tenants", async () => {await Promise.all([30, 0, 20, 10].map((start) =>- trackTraceBatchActivity("project", [event("trace", start)]),+ trackTraceBatchActivity("project", [+ event("trace", start, 100 + start),+ event("trace", start, 200 + start),+ ]),),);const state = JSON.parse((await client().hget(stateKey, member("project", "trace")))!,);- expect(state).toMatchObject({ minStart: 0, maxStart: 30 });+ expect(state).toMatchObject({+ minStart: 0,+ maxStart: 30,+ eventUpdateCount: 8,+ serializedEventBytes: 1_320,+ });await makeDue("project", "trace");- await trackTraceBatchActivity("project", [event("trace", 15)]);+ await trackTraceBatchActivity("project", [+ event("trace", 15, 25),+ event("trace", 15, 25),+ ]);+ expect(+ JSON.parse((await client().hget(stateKey, member("project", "trace")))!),+ ).toMatchObject({ eventUpdateCount: 10, serializedEventBytes: 1_370 });expect(Number(await client().zscore(dueKey, member("project", "trace"))),).toBeGreaterThan(Date.now() + 590_000);- await trackTraceBatchActivity("other", [event("trace", 100)]);+ await trackTraceBatchActivity("other", [event("trace", 100, 7)]);expect(JSON.parse((await client().hget(stateKey, member("other", "trace")))!),- ).toMatchObject({ minStart: 100, maxStart: 100 });+ ).toMatchObject({+ minStart: 100,+ maxStart: 100,+ eventUpdateCount: 1,+ serializedEventBytes: 7,+ });await runner().processBatch();expect(await queue.getWaitingCount()).toBe(0);+ await makeDue("project", "trace");+ await makeDue("other", "trace");+ await runner().processBatch();+ for (const [count, bytes] of [+ [10, 1_370],+ [1, 7],+ ]) {+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.estimated_event_update_count",+ count,+ { scope: "trace", strategy: "project" },+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.estimated_serialized_event_bytes",+ bytes,+ { scope: "trace", strategy: "project" },+ );+ }});it.each([@@ -855,9 +897,14 @@ describe("trace micro-batch scheduling with Redis", () => {);await trackTraceBatchActivity("project",- traces.map((traceId) => event(traceId)),+ traces.map((traceId, index) => event(traceId, 1_000_000, index + 1)),);- await trackTraceBatchActivity("other", [event("trace-0")]);+ await trackTraceBatchActivity("project", [+ event("trace-0", 1_000_000, 5),+ ]);+ await trackTraceBatchActivity("other", [+ event("trace-0", 1_000_000, 10_000),+ ]);await client().zadd(dueKey,...traces.flatMap((traceId, index) => [@@ -917,11 +964,158 @@ describe("trace micro-batch scheduling with Redis", () => {.map(([, value]) => value).sort((a, b) => Number(a) - Number(b)),).toEqual(projectCounts);+ for (const [metric, estimate] of [+ ["estimated_event_update_count", "count"],+ ["estimated_serialized_event_bytes", "bytes"],+ ] as const) {+ const expected = add.mock.calls.map(([, job]) =>+ job.payload.traces.reduce((total, trace) => {+ if (trace.projectId === "other")+ return total + (estimate === "count" ? 1 : 10_000);+ const index = Number(trace.traceId.replace("trace-", ""));+ const estimates =+ index === 0+ ? { count: 2, bytes: 6 }+ : { count: 1, bytes: index + 1 };+ return total + estimates[estimate];+ }, 0),+ );+ expect(+ vi+ .mocked(recordDistribution)+ .mock.calls.filter(+ ([name, , tags]) =>+ name === `langfuse.trace_batch.${metric}` &&+ tags?.scope === "batch",+ )+ .map(([, value, tags]) => ({ value, tags })),+ ).toEqual(+ expected.map((value) => ({+ value,+ tags: { scope: "batch", strategy: "project" },+ })),+ );+ }+ for (const [, job] of add.mock.calls) {+ for (const trace of job.payload.traces) {+ expect(Object.keys(trace).sort()).toEqual([+ "maxStart",+ "minStart",+ "projectId",+ "revision",+ "traceId",+ ]);+ }+ }await runner().processBatch();expect(await queue.getWaitingCount()).toBe(sizes.length);},);+ it("keeps legacy estimates unknown through updates and restores them after dispatch", async () => {+ const legacyStates = [+ {},+ { eventUpdateCount: 4 },+ { serializedEventBytes: 400 },+ ];+ const legacyIds = legacyStates.map((_, index) => `legacy-${index}`);+ for (const [index, estimates] of legacyStates.entries()) {+ await client().hset(+ stateKey,+ member("project", legacyIds[index]),+ JSON.stringify({+ minStart: 1_000_000,+ maxStart: 1_000_000,+ revision: randomUUID(),+ ...estimates,+ }),+ );+ }+ await trackTraceBatchActivity("project", [+ ...legacyIds.map((traceId) => event(traceId, 1_000_001, 20)),+ event("known", 1_000_000, 30),+ ]);+ await makeDue("project", ...legacyIds, "known");+ const dispatcher = runner();+ await dispatcher.processBatch();+ const [job] = await queue.getJobs(["wait"]);+ expect(+ job.data.payload.traces.map(({ traceId }) => traceId).sort(),+ ).toEqual(["known", ...legacyIds].sort());+ const estimateSamples = vi+ .mocked(recordDistribution)+ .mock.calls.filter(([name]) =>+ name.startsWith("langfuse.trace_batch.estimated_"),+ );+ expect(estimateSamples).toHaveLength(2);+ expect(estimateSamples).toEqual(+ expect.arrayContaining([+ [+ "langfuse.trace_batch.estimated_event_update_count",+ 1,+ { scope: "trace", strategy: "project" },+ ],+ [+ "langfuse.trace_batch.estimated_serialized_event_bytes",+ 30,+ { scope: "trace", strategy: "project" },+ ],+ ]),+ );+ for (const [scope, count] of [+ ["trace", 3],+ ["batch", 1],+ ] as const) {+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(+ ([name, , tags]) =>+ name === "langfuse.trace_batch.estimates_unavailable" &&+ tags?.scope === scope &&+ tags?.strategy === "project",+ )+ .reduce((total, [, value]) => total + Number(value), 0),+ ).toBe(count);+ }+ expect(await client().hlen(stateKey)).toBe(0);++ vi.clearAllMocks();+ await trackTraceBatchActivity(+ "project",+ legacyIds.map((traceId) => event(traceId, 1_000_002, 20)),+ );+ for (const traceId of legacyIds) {+ expect(+ JSON.parse(+ (await client().hget(stateKey, member("project", traceId)))!,+ ),+ ).toMatchObject({+ eventUpdateCount: 1,+ serializedEventBytes: 20,+ });+ }+ await makeDue("project", ...legacyIds);+ await dispatcher.processBatch();+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.estimated_event_update_count",+ 3,+ { scope: "batch", strategy: "project" },+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.estimated_serialized_event_bytes",+ 60,+ { scope: "batch", strategy: "project" },+ );+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.some(+ ([name]) => name === "langfuse.trace_batch.estimates_unavailable",+ ),+ ).toBe(false);+ });+it.each(["project", "locality"] as const)("dispatches the complete due cohort across hydration chunks with %s",async (strategy) => {diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 1f4f235732..6eb79a6354 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -35,12 +35,12 @@ processing, and the shared queue accessor, including admin queue inspection.- `{trace-batch}:due`: sorted set, with JSON `[projectId, traceId]` members andRedis server time plus idle delay as the score.- `{trace-batch}:state`: hash with the same members and JSON- `{minStart, maxStart, revision}` values. Bounds describe observed event start+ `{minStart, maxStart, revision, eventUpdateCount, serializedEventBytes}` values. Bounds describe observed event starttimes; revision protects newer ingestion from an older dispatch acknowledgment.- `{trace-batch}:dispatcher`: renewable dispatcher lease.-No event payloads, per-trace byte counts or observation counts are stored here.-Atomic Lua updates refresh bounds, revision and readiness. Each admitted intake+No event payloads are stored here. Atomic Lua updates accumulate estimates and+refresh bounds, revision and readiness. Each admitted intakesets a shared absolute expiry on both pending keys with `PEXPIREAT`, preservingany later existing expiry using `PEXPIRETIME` (Redis 7+). With no further intake,both keys expire after the retention period even if every application stops;@@ -58,6 +58,31 @@ to the configured trace cap; project leftovers can share a job. The dispatcherenqueues before acknowledging matching revisions, so concurrent intake stayspending and failed enqueueing can be retried. Stable job IDs limit duplicates.+### Pending-window size estimates++Accepted direct-v4 writes contribute their final serialized UTF-8 event size after+field overflow handling, reusing the writer's existing `event_bytes` calculation.+The accounting field itself is excluded. Acceptance means admission to the writer,+not confirmation of a ClickHouse flush. `eventUpdateCount` counts accepted updates,+including retries and repeated observation versions; neither counter represents+unique observations or a trace's lifetime total. Deletion after enqueue, readiness+and two-hour retention are unchanged. Later arrivals start another pending window.+Entries missing either counter remain unknown until dispatch or expiry removes them.++The dispatcher emits `estimated_event_update_count` and+`estimated_serialized_event_bytes` distributions under `langfuse.trace_batch`, tagged+with `scope:trace|batch` and `strategy`. Batch totals require known estimates for+every member; `estimates_unavailable` counts unknown traces and affected batches.+Estimates stay out of queue payloads, job IDs and batch selection. They do not cap+work or change oversized-trace handling. Dispatch retries can emit another sample.++Serialized event bytes differ from the reader's `io_metadata_bytes` metric: they+include the serialized event fields and JSON encoding, and do not represent RAM,+network transfer or ClickHouse scan bytes. After rollout, collect a stable hour+with unchanged sampling and batching settings; compare trace/batch distributions,+unknown-estimate counts, ingestion latency and Redis command rate/CPU/memory with+the baseline before choosing any size-aware batching policy.+The worker streams full `events_full` payloads and records batch observation,trace, project and logical payload-byte metrics. The `input_bytes`, `output_bytes`and `metadata_bytes` distributions under `langfuse.trace_batch` each report onediff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex 7c3588cc21..20f83dfe58 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -33,6 +33,7 @@ export type TraceBatchStrategy = "project" | "locality";export type PendingTrace = {member: string;due: number;+ estimates?: { eventUpdateCount: number; serializedEventBytes: number };trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];};@@ -451,18 +452,29 @@ const TRACK_SCRIPT = `${EXPIRE_PENDING_SCRIPT}local clock = redis.call('TIME')local now = clock[1] * 1000 + math.floor(clock[2] / 1000)- for i = 3, #ARGV, 4 do+ for i = 3, #ARGV, 6 dolocal member = ARGV[i]local minStart = tonumber(ARGV[i + 1])local maxStart = tonumber(ARGV[i + 2])+ local eventUpdateCount = tonumber(ARGV[i + 4])+ local serializedEventBytes = tonumber(ARGV[i + 5])local previous = redis.call('HGET', KEYS[2], member)if previous thenlocal state = cjson.decode(previous)minStart = math.min(minStart, state.minStart)maxStart = math.max(maxStart, state.maxStart)+ if state.eventUpdateCount and state.serializedEventBytes then+ eventUpdateCount = eventUpdateCount + state.eventUpdateCount+ serializedEventBytes = serializedEventBytes + state.serializedEventBytes+ else+ -- Missing history cannot be reconstructed from a later update.+ eventUpdateCount = nil+ serializedEventBytes = nil+ endendredis.call('HSET', KEYS[2], member, cjson.encode({- minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3]+ minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3],+ eventUpdateCount = eventUpdateCount, serializedEventBytes = serializedEventBytes}))redis.call('ZADD', KEYS[1], now + tonumber(ARGV[1]), member)end@@ -531,7 +543,11 @@ const ACKNOWLEDGE_SCRIPT = `export async function trackTraceBatchActivity(projectId: string,- events: { traceId: string; startTimeISO: string }[],+ events: {+ traceId: string;+ startTimeISO: string;+ serializedEventBytes: number;+ }[],): Promise<void> {if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||@@ -539,7 +555,15 @@ export async function trackTraceBatchActivity()return;- const bounds = new Map<string, { minStart: number; maxStart: number }>();+ const bounds = new Map<+ string,+ {+ minStart: number;+ maxStart: number;+ eventUpdateCount: number;+ serializedEventBytes: number;+ }+ >();for (const event of events) {const start = Date.parse(event.startTimeISO);if (!Number.isFinite(start)) continue;@@ -547,6 +571,9 @@ export async function trackTraceBatchActivity(bounds.set(event.traceId, {minStart: Math.min(previous?.minStart ?? start, start),maxStart: Math.max(previous?.maxStart ?? start, start),+ eventUpdateCount: (previous?.eventUpdateCount ?? 0) + 1,+ serializedEventBytes:+ (previous?.serializedEventBytes ?? 0) + event.serializedEventBytes,});}if (bounds.size === 0) return;@@ -593,6 +620,8 @@ export async function trackTraceBatchActivity(state.maxStart,// Unique even after dispatch deletes and a later arrival recreates state.randomUUID(),+ state.eventUpdateCount,+ state.serializedEventBytes,]),),);@@ -765,7 +794,26 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {recordIncrement("langfuse.trace_batch.dispatched_traces", batch.length, {batch_kind: batch.length === 1 ? "singleton" : "multi",});+ let estimatedEventUpdateCount = 0;+ let estimatedSerializedEventBytes = 0;+ let unavailableEstimates = 0;for (const entry of batch) {+ if (entry.estimates) {+ estimatedEventUpdateCount += entry.estimates.eventUpdateCount;+ estimatedSerializedEventBytes += entry.estimates.serializedEventBytes;+ recordDistribution(+ "langfuse.trace_batch.estimated_event_update_count",+ entry.estimates.eventUpdateCount,+ { scope: "trace", strategy },+ );+ recordDistribution(+ "langfuse.trace_batch.estimated_serialized_event_bytes",+ entry.estimates.serializedEventBytes,+ { scope: "trace", strategy },+ );+ } else {+ unavailableEstimates++;+ }recordDistribution("langfuse.trace_batch.observed_start_span_ms",entry.trace.maxStart - entry.trace.minStart,@@ -776,6 +824,31 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {Math.max(0, Date.now() - entry.due),);}+ if (unavailableEstimates === 0) {+ recordDistribution(+ "langfuse.trace_batch.estimated_event_update_count",+ estimatedEventUpdateCount,+ { scope: "batch", strategy },+ );+ recordDistribution(+ "langfuse.trace_batch.estimated_serialized_event_bytes",+ estimatedSerializedEventBytes,+ { scope: "batch", strategy },+ );+ } else {+ recordIncrement(+ "langfuse.trace_batch.estimates_unavailable",+ unavailableEstimates,+ {+ scope: "trace",+ strategy,+ },+ );+ recordIncrement("langfuse.trace_batch.estimates_unavailable", 1, {+ scope: "batch",+ strategy,+ });+ }const removed = Number(await redis!.eval(ACKNOWLEDGE_SCRIPT,@@ -818,11 +891,22 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {if (this.stopping) return;const member = hydrated[i];const [projectId, traceId] = JSON.parse(member) as [string, string];+ const state = JSON.parse(hydrated[i + 1]);hydratedCandidates.push({member,due: Number(hydrated[i + 2]),+ estimates:+ Number.isSafeInteger(state.eventUpdateCount) &&+ state.eventUpdateCount >= 0 &&+ Number.isSafeInteger(state.serializedEventBytes) &&+ state.serializedEventBytes >= 0+ ? {+ eventUpdateCount: state.eventUpdateCount,+ serializedEventBytes: state.serializedEventBytes,+ }+ : undefined,trace: TraceBatchTraceSchema.parse({- ...JSON.parse(hydrated[i + 1]),+ ...state,projectId,traceId,}),diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.tsindex 1b65d2ce64..6a1e388ec4 100644--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts@@ -92,7 +92,7 @@ describe("direct-v4 trace batch tracking", () => {"createEventRecord",).mockImplementation(async (input) => {if (input.traceId === "conversion-failed") throw new Error("invalid");- return { span_id: input.spanId } as Awaited<+ return { span_id: input.spanId, event_bytes: 10_000 } as Awaited<ReturnType<IngestionService["createEventRecord"]>>;});@@ -100,6 +100,7 @@ describe("direct-v4 trace batch tracking", () => {.spyOn(IngestionService.prototype, "writeEventRecord").mockImplementation(async (record) => {if (record.span_id === "third") throw new Error("write rejected");+ return record.span_id === "first" ? 101 : 202;});const processor = otelIngestionQueueProcessorBuilder(false);const job = {@@ -126,9 +127,11 @@ describe("direct-v4 trace batch tracking", () => {expect(write).toHaveBeenCalledTimes(3);expect(trackTraceBatchActivity).toHaveBeenCalledExactlyOnceWith("project",- inputs- .slice(0, 2)- .map(({ traceId, startTimeISO }) => ({ traceId, startTimeISO })),+ inputs.slice(0, 2).map(({ traceId, startTimeISO }, index) => ({+ traceId,+ startTimeISO,+ serializedEventBytes: index === 0 ? 101 : 202,+ })),);expect(recordDistribution).toHaveBeenCalledWith("langfuse.trace_batch.ingestion_trace_count",diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.tsindex 3b5045a629..876d442a83 100644--- a/worker/src/queues/otelIngestionQueue.ts+++ b/worker/src/queues/otelIngestionQueue.ts@@ -832,7 +832,11 @@ export const otelIngestionQueueProcessorBuilder = (? createObservationEvalSchedulerDeps(): null;- const traceBatchEvents: { traceId: string; startTimeISO: string }[] = [];+ const traceBatchEvents: {+ traceId: string;+ startTimeISO: string;+ serializedEventBytes: number;+ }[] = [];await Promise.all(// Process each event independently@@ -883,7 +887,8 @@ export const otelIngestionQueueProcessorBuilder = (// Step 3: Write to events table (independent of eval scheduling)if (shouldWriteToEventsTable) {try {- await ingestionService.writeEventRecord(eventRecord);+ const serializedEventBytes =+ await ingestionService.writeEventRecord(eventRecord);if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED === "true"@@ -891,6 +896,7 @@ export const otelIngestionQueueProcessorBuilder = (traceBatchEvents.push({traceId: eventInput.traceId,startTimeISO: eventInput.startTimeISO,+ serializedEventBytes,});}} catch (error) {diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 01a44ecfdd..e4b82892f3 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -500,16 +500,17 @@ export class IngestionService {* supplied record is not mutated.** @param eventRecord - The event record to write+ * @returns The serialized byte size of the accepted event record*/public async writeEventRecord(eventRecord: EventRecordInsertType,- ): Promise<void> {- const persistedRecord = await applyObservationFieldOverflow(eventRecord);-- this.clickHouseWriter.addToQueue(- TableName.EventsFull,- withSerializedEventByteLength(persistedRecord),+ ): Promise<number> {+ const persistedRecord = withSerializedEventByteLength(+ await applyObservationFieldOverflow(eventRecord),);++ this.clickHouseWriter.addToQueue(TableName.EventsFull, persistedRecord);+ return persistedRecord.event_bytes;}private async processDatasetRunItemEventList(params: {diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.tsindex 9a9b7ef67e..af46f770ad 100644--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts@@ -77,7 +77,7 @@ describe("IngestionService unit tests", () => {expect(eventRecord.event_bytes).toBe(rawOtelSpanBytes);- await ingestionService.writeEventRecord(eventRecord);+ const acceptedBytes = await ingestionService.writeEventRecord(eventRecord);expect(addToQueue).toHaveBeenCalledOnce();const queuedRecord = addToQueue.mock.calls[0]?.[1];@@ -89,6 +89,7 @@ describe("IngestionService unit tests", () => {Buffer.byteLength(JSON.stringify(eventWithoutSize), "utf8"),);expect(eventBytes).toBeLessThan(rawOtelSpanBytes);+ expect(acceptedBytes).toBe(eventBytes);});it("overflows only the direct events_full copy and preserves the enriched record", async () => {@@ -128,7 +129,7 @@ describe("IngestionService unit tests", () => {],});- await ingestionService.writeEventRecord(eventRecord);+ const acceptedBytes = await ingestionService.writeEventRecord(eventRecord);expect(mocks.applyObservationFieldOverflow).toHaveBeenCalledWith(eventRecord,@@ -141,6 +142,12 @@ describe("IngestionService unit tests", () => {metadata_names: ["keep", "large.nested"],metadata_values: ["small", expect.stringContaining("metadata-media")],});+ const { event_bytes: eventBytes, ...eventWithoutSize } =+ addToQueue.mock.calls[0]![1];+ expect(acceptedBytes).toBe(eventBytes);+ expect(acceptedBytes).toBe(+ Buffer.byteLength(JSON.stringify(eventWithoutSize), "utf8"),+ );});it("promotes provided usage and cost for model-less direct events", async () => {