🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(trace-batching): add capacity experiment telemetry (#17614)
Lysander Kiesel6e9f8ebdff15
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex 2a8aea8f41..bbab71cfc3 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -277,7 +277,7 @@ export async function upsertClickhouse<}export async function* queryClickhouseStream<T>(- opts: ClickhouseQueryOpts,+ opts: ClickhouseQueryOpts & { queryId?: string },): AsyncGenerator<T> {if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);const normalizedTags = normalizeClickHouseQueryTags(opts.tags);@@ -288,7 +288,7 @@ export async function* queryClickhouseStream<T>(// Client-generated so failures before/without a response still carry a// query_id on errors and spans; system.query_log stays pollable by id.- const queryId = randomUUID();+ const queryId = opts.queryId ?? randomUUID();try {setSpanQueryAttributes(span, opts.query);diff --git a/packages/shared/src/server/repositories/trace-batch.test.ts b/packages/shared/src/server/repositories/trace-batch.test.tsindex db8cd0b39a..619684adea 100644--- a/packages/shared/src/server/repositories/trace-batch.test.ts+++ b/packages/shared/src/server/repositories/trace-batch.test.ts@@ -82,13 +82,19 @@ describe("trace batch query controls", () => {requestedSecondRow = false;const stream = getTraceBatchEventStream({ traces: members },- { maxThreads: 1, maxBlockSize, experimentId: "arm-b" },+ {+ maxThreads: 1,+ maxBlockSize,+ experimentId: "arm-b",+ queryId: "read-attempt",+ },);expect(await stream.next()).toEqual({ value: row, done: false });expect(requestedSecondRow).toBe(false);const sent = vi.mocked(queryClickhouseStream).mock.lastCall![0];expect(sent).toEqual({...baseline,+ queryId: "read-attempt",tags: { ...baseline.tags, experimentId: "arm-b" },clickhouseSettings: {...baseline.clickhouseSettings,diff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.tsindex 28c0b862f2..56e00e6b7d 100644--- a/packages/shared/src/server/repositories/trace-batch.ts+++ b/packages/shared/src/server/repositories/trace-batch.ts@@ -196,6 +196,7 @@ export async function* getTraceBatchEventStream(maxThreads?: number;maxBlockSize?: number;experimentId?: string;+ queryId?: string;} = {},): AsyncGenerator<TraceBatchEventRow> {if (props.traces.length === 0) return;@@ -203,6 +204,7 @@ export async function* getTraceBatchEventStream(const { query, params, projectIds } = buildTraceBatchEventQuery(props);yield* queryClickhouseStream<TraceBatchEventRow>({query,+ queryId: options.queryId,params,useMultipartParamsAuto: true,tags: {diff --git a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.tsindex 9ff6c74a2c..c71b0ef072 100644--- a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts+++ b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts@@ -7,6 +7,7 @@ import {ClickHouseResourceError,} from "@langfuse/shared/src/server";import { fail } from "assert";+import { randomUUID } from "node:crypto";const QUERY_ID_PATTERN =/\[query_id: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\]/;@@ -124,8 +125,11 @@ describe("ClickHouse Resource Error Handling", () => {});it("should stream successful queries", async () => {+ const queryId = randomUUID();const generator = queryClickhouseStream({- query: "SELECT number FROM system.numbers LIMIT 3",+ query:+ "SELECT number, queryID() AS query_id FROM system.numbers LIMIT 3",+ queryId,});const results = [];@@ -137,6 +141,7 @@ describe("ClickHouse Resource Error Handling", () => {expect(Array.isArray(results)).toBe(true);expect(results.length).toBe(3);expect(results[0]).toHaveProperty("number");+ expect(results[0]).toHaveProperty("query_id", queryId);});});diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex c0310a725f..99e2327963 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -22,6 +22,9 @@independently default-off and cloud-gated. Do not expose these PoC controls inlocal or production env templates. Reader query controls are independent oflocality selection; logs must preserve separate input/output/metadata metrics.+ `src/features/traceBatching/TraceBatchMetricsRunner.ts` collects bounded queue+ and Redis snapshots independently of dispatch/consumption when either role is+ enabled. Global snapshot gauges must not be summed across worker reporters.- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.- Service layer: `src/services/*`- Rust addon (`@langfuse/native`): telemetry init and the startup hello call livediff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex 6536c90455..ef9e37e4dd 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -18,6 +18,7 @@ import {QueueJobs,QueueName,recordDistribution,+ recordGauge,recordIncrement,redis,TraceBatchEventSchema,@@ -26,6 +27,7 @@ import {} from "@langfuse/shared/src/server";import * as shared from "@langfuse/shared/src/server";import { env } from "../env";+import { TraceBatchMetricsRunner } from "../features/traceBatching/TraceBatchMetricsRunner";import {prepareLocalityPartials,selectTraceBatches,@@ -46,6 +48,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {keyPrefix: `trace-batch-test-${randomUUID()}:`,}),recordDistribution: vi.fn(),+ recordGauge: vi.fn(),recordIncrement: vi.fn(),getS3EventStorageClient: vi.fn(),};@@ -415,6 +418,8 @@ describe("trace micro-batch scheduling with Redis", () => {const originalEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;+ const originalConsumerEnabled =+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED;const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;const originalSamplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;const originalStrategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;@@ -456,6 +461,7 @@ describe("trace micro-batch scheduling with Redis", () => {beforeEach(async () => {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "true";+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED = "false";env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;@@ -480,6 +486,7 @@ describe("trace micro-batch scheduling with Redis", () => {env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = originalEnabled;env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED = originalConsumerEnabled;env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = originalSamplingRate;env.LANGFUSE_TRACE_BATCH_STRATEGY = originalStrategy;@@ -493,6 +500,101 @@ describe("trace micro-batch scheduling with Redis", () => {});afterAll(() => client().disconnect());+ it("samples prefixed map memory and paused/delayed queue work, then reports drained zeros", async () => {+ const now = Date.now();+ const clock = vi.spyOn(Date, "now").mockReturnValue(now);+ await trackTraceBatchActivity("project", [event("trace")]);+ const data = {+ id: randomUUID(),+ name: QueueJobs.TraceBatch,+ timestamp: new Date(now),+ payload: { traces: [] },+ };+ const waiting = await queue.add(QueueJobs.TraceBatch, data, {+ timestamp: now - 5_000,+ });+ await queue.add(QueueJobs.TraceBatch, data, { timestamp: now - 1_000 });+ await queue.add(QueueJobs.TraceBatch, data, { delay: 120_000 });+ await queue.pause();+ // Age collection must not fetch or parse the batch payload.+ await connection.hset(queue.toKey(waiting.id!), "data", "invalid JSON");+ const metrics = new TraceBatchMetricsRunner();+ const evaluate = vi.spyOn(client(), "eval");+ await metrics["execute"]();++ for (const key of ["due", "state"]) {+ expect(recordGauge).toHaveBeenCalledWith(+ "langfuse.trace_batch.redis_key_entries",+ 1,+ { key, unit: "records" },+ );+ const sample = vi+ .mocked(recordGauge)+ .mock.calls.find(+ ([name, , tags]) =>+ name === "langfuse.trace_batch.redis_key_bytes" &&+ tags?.key === key,+ );+ expect(Number(sample?.[1])).toBeGreaterThan(0);+ }+ for (const type of ["waiting", "delayed"]) {+ expect(recordGauge).toHaveBeenCalledWith(+ "langfuse.trace_batch.queue_depth",+ type === "waiting" ? 2 : 1,+ { type, unit: "records" },+ );+ }+ expect(recordGauge).toHaveBeenCalledWith(+ "langfuse.trace_batch.queue_waiting_head_age_ms",+ 5_000,+ { unit: "milliseconds" },+ );++ clock.mockReturnValue(now + 30_000);+ await metrics["execute"]();+ expect(evaluate).toHaveBeenCalledTimes(1);+ await client().del(dueKey, stateKey);+ await queue.drain(true);+ vi.mocked(recordGauge).mockClear();+ clock.mockReturnValue(now + 60_000);+ await metrics["execute"]();+ expect(evaluate).toHaveBeenCalledTimes(2);+ expect(recordGauge).toHaveBeenCalledWith(+ "langfuse.trace_batch.active_reads",+ 0,+ );+ expect(vi.mocked(recordGauge).mock.calls).toHaveLength(10);+ expect(+ vi.mocked(recordGauge).mock.calls.every(([, value]) => value === 0),+ ).toBe(true);+ });++ it("awaits failed telemetry collections while preserving independent queue measurements", async () => {+ const metrics = new TraceBatchMetricsRunner();+ const pendingMemory = Promise.withResolvers<number[]>();+ vi.spyOn(client(), "eval").mockImplementationOnce(+ () => pendingMemory.promise,+ );+ vi.spyOn(queue, "getJobCounts").mockRejectedValueOnce(+ new Error("queue unavailable"),+ );+ let completed = false;+ const collection = metrics["execute"]().then(() => {+ completed = true;+ });+ await vi.waitFor(() =>+ expect(recordGauge).toHaveBeenCalledWith(+ "langfuse.trace_batch.queue_waiting_head_age_ms",+ 0,+ { unit: "milliseconds" },+ ),+ );+ expect(completed).toBe(false);+ pendingMemory.reject(new Error("memory unavailable"));+ await collection;+ expect(vi.mocked(recordGauge).mock.calls).toHaveLength(2);+ });+it("dispatches with locality selection and records bounded selector measurements", async () => {env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";const minute = 60_000;@@ -563,6 +665,11 @@ describe("trace micro-batch scheduling with Redis", () => {const lock = vi.spyOn(client(), "set");await trackTraceBatchActivity("project", [event("trace")]);await runner()["execute"]();+ const metrics = new TraceBatchMetricsRunner();+ await metrics["execute"]();+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "false";+ await metrics["execute"]();expect(evaluate).not.toHaveBeenCalled();expect(lock).not.toHaveBeenCalled();expect(await client().exists(dueKey, stateKey)).toBe(0);@@ -801,8 +908,8 @@ describe("trace micro-batch scheduling with Redis", () => {"project",traceIds.flatMap((id) => [event(id, 100),- event(id, 200),- event(id, 150),+ event(id, 200, 200),+ event(id, 150, 50),]),);const selected = (await client().hkeys(stateKey)).sort();@@ -820,10 +927,73 @@ describe("trace micro-batch scheduling with Redis", () => {1_000 - expectedCount,{ decision: "excluded" },);+ for (const [stage, count] of [+ ["eligible", 1_000],+ ["sampled", expectedCount],+ ...(expectedCount > 0 ? [["recorded", expectedCount] as const] : []),+ ] as const) {+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.event_updates",+ count * 3,+ { stage },+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.serialized_event_bytes",+ count * 350,+ { stage },+ );+ }previous = selected;}});+ it("counts only acknowledged update volume when tracking fails after a chunk", async () => {+ const evaluate = client().eval.bind(client());+ vi.spyOn(client(), "eval")+ .mockImplementationOnce(evaluate)+ .mockRejectedValueOnce(new Error("tracking connection failed"));++ await expect(+ trackTraceBatchActivity("project", [+ ...Array.from({ length: 1_001 }, (_, index) =>+ event(`trace-${index}`, 100, 7),+ ),+ { ...event("invalid"), startTimeISO: "not-a-date" },+ ]),+ ).resolves.toBeUndefined();++ expect(await client().hlen(stateKey)).toBe(1_000);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.event_updates",+ 1_001,+ { stage: "eligible" },+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.event_updates",+ 1_001,+ { stage: "sampled" },+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.event_updates",+ 1_000,+ { stage: "recorded" },+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.serialized_event_bytes",+ 7_000,+ { stage: "recorded" },+ );+ expect(recordIncrement).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.event_updates",+ 1,+ { stage: "recorded" },+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.tracking_errors",+ 1,+ );+ });+it("preserves min/max across concurrent out-of-order arrivals, resets readiness, and separates tenants", async () => {await Promise.all([30, 0, 20, 10].map((start) =>diff --git a/worker/src/app.ts b/worker/src/app.tsindex f979e0e4c2..1d16cb798c 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -60,6 +60,7 @@ import { experimentCreateQueueProcessor } from "./queues/experimentQueue";import { traceDeleteProcessor } from "./queues/traceDelete";import { traceBatchQueueProcessor } from "./queues/traceBatchQueue";import { TraceBatchDispatcher } from "./features/traceBatching/traceBatching";+import { TraceBatchMetricsRunner } from "./features/traceBatching/TraceBatchMetricsRunner";import { projectDeleteProcessor } from "./queues/projectDelete";import {postHogIntegrationProcessingProcessor,@@ -156,6 +157,16 @@ if (});}+export let traceBatchMetricsRunner: TraceBatchMetricsRunner | null = null;+if (+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&+ (env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED === "true" ||+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED === "true")+) {+ traceBatchMetricsRunner = new TraceBatchMetricsRunner();+ traceBatchMetricsRunner.start();+}+if (env.QUEUE_CONSUMER_TRACE_UPSERT_QUEUE_IS_ENABLED === "true") {// Register workers for all trace upsert queue shardsconst traceUpsertShardNames = TraceUpsertQueue.getShardNames();diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 6eb79a6354..8c2f4cf8d5 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -102,6 +102,91 @@ Queries use response compression and chunked multipart parameters for up torather than accepting partial results. This does not cap observation count,query memory or dispatcher snapshot memory.+## Capacity measurements++The following metrics use the `langfuse.trace_batch` prefix. All additions are+inside the existing cloud experiment paths; normal ingestion with tracking+disabled performs no new aggregation or Redis calls.++| Metric | Kind / tags | Meaning |+| --- | --- | --- |+| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|sampled\|recorded` | Accepted, valid-start-time updates before sampling, after sampling, and in Redis-acknowledged chunks. Repeated updates count again. |+| `read_attempts` | Counter; `outcome:success\|failure\|discard` | One outcome per processor invocation, including retries, validation failures and disabled/expired discards. |+| `read_duration_ms` | Distribution; same outcome tags | Wall-clock processor duration, including failed and discarded attempts. |+| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |+| `failed_read_observation_count`, `failed_read_input_bytes`, `failed_read_output_bytes`, `failed_read_metadata_bytes`, `failed_read_io_metadata_bytes` | Distributions | Partial logical rows/bytes consumed before a stream failure. Separate from successful throughput. |+| `queue_depth` | Gauge; `type:waiting\|active\|delayed\|failed` | Global BullMQ snapshots; waiting includes paused work. |+| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |+| `redis_key_bytes` | Gauge; `key:due\|state` | Estimated memory of each readiness key, including Redis overhead. |+| `redis_key_entries` | Gauge; `key:due\|state` | `ZCARD` / `HLEN` at the memory snapshot. |++The eligible/sampled counters reuse the already-computed serialized size and+per-trace aggregates. Recorded volume is emitted after each acknowledged Lua+chunk: earlier chunks remain counted if a later one fails. A timeout can mean+Redis applied an update without acknowledgement, so the stage difference includes+uncertain outcomes; it does not prove data loss. Writer acceptance still precedes+the ClickHouse flush. No additional serialization is performed.++`TraceBatchMetricsRunner` starts when either the dispatcher or consumer is+enabled in a cloud worker, independently of the generic queue-metrics flag.+It polls the queue every 30 seconds and map memory every 60 seconds per process,+plus collection time. No completed jobs or progressing dispatcher are required.+Queue-age collection reads only the next waiting/paused job IDs and their+timestamp fields; it never fetches or parses batch payloads.+One failed collection does not suppress the others; failed samples are not+reported as zero. Check `langfuse.periodic_runner.completed` and+`last_healthy_timestamp_seconds` with `runner:trace_batch_metrics` for freshness.++Memory collection uses one bounded two-key Lua call with `MEMORY USAGE ...+SAMPLES 5`, preserving the shared key prefix and Redis Cluster slot routing.+Missing keys report zero; there is no keyspace scan or exact full-hash traversal.+Memory sampling estimates storage, not process RSS or event payload size.++Queue/map snapshots are global and can be emitted by several workers: use+max/latest across reporters, never sum duplicates. For map totals, first+deduplicate reporters per `key`, then add the due/state estimates. In contrast,+`active_reads` is local to a worker; distinct live worker series may be summed.+It is emitted on stream start/finish and on each metrics-runner cycle, including+zero when idle, so unchanged long-running reads remain visible.+Instantaneous gauges can miss short-lived peaks. A retried job may re-enter+behind newer jobs, so waiting-head age is a bounded-cost backlog indicator, not+an exact oldest-created-job measurement across the entire queue.++With an experiment ID, start/completion logs include `jobId`, `attempt`,+`queryId` and `experimentId`, with the completion outcome. The per-read UUID is+forwarded to ClickHouse as `query_id` and remains the root correlation ID for+the stream. Discards and validation failures have no query ID. Identifiers are+not metric tags and logs contain no event payloads. Successful batch metrics and+job return values retain their existing meanings.++Completion logs also correlate the requested `batchTraceCount`, distinct+`batchProjectCount`, `eventTimeSpanMs` (outer event-time span, without query+buffering), and `maxTraceSpanMs` with the outcome and `durationMs`. These are+batch-locality proxies, not measured ClickHouse scan costs. `observationCount`,+`foundTraceCount`, `foundProjectCount` and separate `inputBytes`, `outputBytes`,+`metadataBytes` describe rows consumed by that attempt. `partial: true` marks+failed reads, including failures before any row arrives; zero then does not mean+the requested batch was empty. Successful counts cover the completed stream,+not unique ingested events. Discarded/unparsed jobs omit stream counters.+Use these existing logs to compare batch shape and volume across outcomes by+query ID; no payload contents, project-ID lists or per-batch metric tags are added.++For the initial scaling curve, use deterministic trace sampling at 10%, 20%,+50%, then 100% across the same project population. Annotate fixed UTC windows;+pause tracking and drain old readiness/queued work between settings. Warm up at+least 20 minutes and compare a settled hour at each stage, holding batch/query+settings fixed. Plot actual admitted updates/bytes and returned rows/bytes against+CPU, memory and backlog rather than assuming configured percentages equal load.+These additions do not generate amplified reads or change batching, query limits,+late-arrival behavior, queue payloads or retention.++Pair the metrics with per-worker RSS/CPU/event-loop and network telemetry, Redis+primary telemetry, and experiment-filtered ClickHouse query logs. Include failed+attempts in CPU cost, count successful root results once, and verify distributed+CPU-counter accounting before summing query parts. Query memory peaks are not+actual concurrent replica RAM. Validate live metric delivery and units before+raising production load.+## Compare reader settingsEstablish a fresh control measurement after deploying this reader: both armsdiff --git a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts b/worker/src/features/traceBatching/TraceBatchMetricsRunner.tsnew file mode 100644index 0000000000..fd79cbc1ad--- /dev/null+++ b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts@@ -0,0 +1,147 @@+import {+ logger,+ recordGauge,+ redis,+ TraceBatchQueue,+} from "@langfuse/shared/src/server";+import { env } from "../../env";+import { recordTraceBatchActiveReads } from "../../queues/traceBatchQueue";+import { PeriodicRunner } from "../../utils/PeriodicRunner";++const METRIC_PREFIX = "langfuse.trace_batch";+const MEMORY_INTERVAL_MS = 60_000;+// EVAL applies the shared client's key prefix and routes both keys to their+// cluster slot. Raw MEMORY commands do not identify keys in ioredis metadata.+const MEMORY_SCRIPT = `+return {+ redis.call('MEMORY', 'USAGE', KEYS[1], 'SAMPLES', 5) or 0,+ redis.call('ZCARD', KEYS[1]),+ redis.call('MEMORY', 'USAGE', KEYS[2], 'SAMPLES', 5) or 0,+ redis.call('HLEN', KEYS[2])+}+`;++export class TraceBatchMetricsRunner extends PeriodicRunner {+ private lastMemorySampleAt: number | null = null;++ constructor() {+ super("trace_batch_metrics");+ }++ protected get name(): string {+ return "trace-batch-metrics";+ }++ protected get defaultIntervalMs(): number {+ return 30_000;+ }++ protected async execute(): Promise<void> {+ if (+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||+ (env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED !== "true" &&+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED !== "true")+ ) {+ return;+ }++ recordTraceBatchActiveReads();+ const collections = [+ this.collectQueueDepth(),+ this.collectWaitingHeadAge(),+ ];+ const now = Date.now();+ if (+ this.lastMemorySampleAt === null ||+ now - this.lastMemorySampleAt >= MEMORY_INTERVAL_MS+ ) {+ this.lastMemorySampleAt = now;+ collections.push(this.collectMemory());+ }++ // Each collection remains useful if another fails. Never replace a failed+ // measurement with zero or leave issued Redis commands unawaited.+ const results = await Promise.allSettled(collections);+ for (const result of results) {+ if (result.status === "rejected") {+ this.markRunFailed(result.reason);+ logger.error("Trace batch metrics collection failed", {+ error: result.reason,+ });+ }+ }+ }++ private async collectQueueDepth(): Promise<void> {+ const queue = TraceBatchQueue.getInstance();+ if (!queue) throw new Error("Trace batch queue is unavailable");+ const counts = await queue.getJobCounts(+ "waiting",+ "paused",+ "active",+ "delayed",+ "failed",+ );+ const depths = {+ waiting: (counts.waiting ?? 0) + (counts.paused ?? 0),+ active: counts.active ?? 0,+ delayed: counts.delayed ?? 0,+ failed: counts.failed ?? 0,+ };+ for (const [type, depth] of Object.entries(depths)) {+ recordGauge(`${METRIC_PREFIX}.queue_depth`, depth, {+ type,+ unit: "records",+ });+ }+ }++ private async collectWaitingHeadAge(): Promise<void> {+ const queue = TraceBatchQueue.getInstance();+ if (!queue) throw new Error("Trace batch queue is unavailable");+ // BullMQ returns the next FIFO entry in each of waiting and paused. A retry+ // can re-enter behind newer jobs, so this is not the oldest creation time+ // anywhere in the queue. Jobs can disappear between ID and hash reads.+ const ids = await queue.getRanges(["wait", "paused"], 0, 0, true);+ const client = await queue.client;+ const timestamps = await Promise.all(+ ids.map((id) => client.hget(queue.toKey(id), "timestamp")),+ );+ const now = Date.now();+ const ages = timestamps.flatMap((timestamp) =>+ timestamp !== null && Number.isFinite(Number(timestamp))+ ? [now - Number(timestamp)]+ : [],+ );+ recordGauge(+ `${METRIC_PREFIX}.queue_waiting_head_age_ms`,+ Math.max(0, ...ages),+ { unit: "milliseconds" },+ );+ }++ private async collectMemory(): Promise<void> {+ if (!redis) throw new Error("Trace batch metrics require Redis");+ const [dueBytes, dueEntries, stateBytes, stateEntries] = (await redis.eval(+ MEMORY_SCRIPT,+ 2,+ "{trace-batch}:due",+ "{trace-batch}:state",+ )) as number[];+ // These are global snapshots emitted by each eligible worker. Aggregate+ // duplicate reporters with max/latest, never sum their identical gauges.+ for (const { key, bytes, entries } of [+ { key: "due", bytes: dueBytes, entries: dueEntries },+ { key: "state", bytes: stateBytes, entries: stateEntries },+ ]) {+ recordGauge(`${METRIC_PREFIX}.redis_key_bytes`, bytes, {+ key,+ unit: "bytes",+ });+ recordGauge(`${METRIC_PREFIX}.redis_key_entries`, entries, {+ key,+ unit: "records",+ });+ }+ }+}diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex 20f83dfe58..89b548bc4f 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -541,6 +541,27 @@ const ACKNOWLEDGE_SCRIPT = `return removed`;+function recordTrackingVolume(+ stage: "eligible" | "sampled" | "recorded",+ entries: Iterable<+ readonly [+ string,+ { eventUpdateCount: number; serializedEventBytes: number },+ ]+ >,+): void {+ let updates = 0;+ let bytes = 0;+ for (const [, state] of entries) {+ updates += state.eventUpdateCount;+ bytes += state.serializedEventBytes;+ }+ recordIncrement("langfuse.trace_batch.event_updates", updates, { stage });+ recordIncrement("langfuse.trace_batch.serialized_event_bytes", bytes, {+ stage,+ });+}+export async function trackTraceBatchActivity(projectId: string,events: {@@ -579,6 +600,7 @@ export async function trackTraceBatchActivity(if (bounds.size === 0) return;try {+ recordTrackingVolume("eligible", bounds);const samplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;// Sample by trace ID so later observations and retries keep the same decision.// Dispatcher and consumer process admitted work without resampling.@@ -601,6 +623,7 @@ export async function trackTraceBatchActivity(bounds.size - entries.length,{ decision: "excluded" },);+ recordTrackingVolume("sampled", entries);if (entries.length === 0) return;if (!redis) throw new Error("Trace batching requires Redis");// Bounded scripts keep ingestion from monopolizing the global Redis slot.@@ -625,6 +648,8 @@ export async function trackTraceBatchActivity(]),),);+ // Count only acknowledged chunks; a timeout can leave Redis's outcome unknown.+ recordTrackingVolume("recorded", chunk);recordIncrement("langfuse.trace_batch.expired_traces", expired);recordIncrement("langfuse.trace_batch.tracked_traces", chunk.length);}diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex 6edd7b65ec..0f42729d51 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -6,19 +6,27 @@ import {QueueJobs,type QueueName,recordDistribution,+ recordGauge,+ recordIncrement,type TQueueJobTypes,} from "@langfuse/shared/src/server";import { env } from "../../env";-import { traceBatchQueueProcessor } from "../traceBatchQueue";+import {+ recordTraceBatchActiveReads,+ traceBatchQueueProcessor,+} from "../traceBatchQueue";vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),getTraceBatchEventStream: vi.fn(),recordDistribution: vi.fn(),+ recordGauge: vi.fn(),+ recordIncrement: vi.fn(),}));const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;+const originalExperimentId = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;beforeEach(() => {env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";@@ -26,6 +34,7 @@ beforeEach(() => {afterEach(() => {env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = originalExperimentId;vi.restoreAllMocks();vi.clearAllMocks();});@@ -41,7 +50,17 @@ describe("trace batch queue", () => {discarded: "not_cloud",});expect(getTraceBatchEventStream).not.toHaveBeenCalled();- expect(recordDistribution).not.toHaveBeenCalled();+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.read_attempts",+ 1,+ { outcome: "discard" },+ );+ expect(recordDistribution).toHaveBeenCalledExactlyOnceWith(+ "langfuse.trace_batch.read_duration_ms",+ expect.any(Number),+ { outcome: "discard" },+ );+ expect(recordGauge).not.toHaveBeenCalled();expect(job.opts.removeOnComplete).toBe(true);});@@ -71,7 +90,19 @@ describe("trace batch queue", () => {});}expect(getTraceBatchEventStream).not.toHaveBeenCalled();- expect(recordDistribution).not.toHaveBeenCalled();+ expect(vi.mocked(recordDistribution).mock.calls).toEqual([+ [+ "langfuse.trace_batch.read_duration_ms",+ expect.any(Number),+ { outcome: "discard" },+ ],+ [+ "langfuse.trace_batch.read_duration_ms",+ expect.any(Number),+ { outcome: "discard" },+ ],+ ]);+ expect(recordGauge).not.toHaveBeenCalled();expect(job.opts.removeOnComplete).toBe(true);});@@ -97,7 +128,7 @@ describe("trace batch queue", () => {name: "generation",input: "input",output: "output",- metadata: {},+ metadata: { é: "界🙂" },tool_definitions: {},tool_calls: [],tool_call_names: [],@@ -113,9 +144,18 @@ describe("trace batch queue", () => {maxStart: 2_000,revision: "r",},+ {+ projectId: "other",+ traceId: "trace",+ minStart: 9_000,+ maxStart: 12_000,+ revision: "r",+ },],};const job = {+ id: "batch-job",+ attemptsMade: 1,data: {id: "batch",name: QueueJobs.TraceBatch,@@ -131,6 +171,7 @@ describe("trace batch queue", () => {maxThreads: 1,maxBlockSize: 512,experimentId: "arm-b",+ queryId: expect.any(String),});expect(log).toHaveBeenCalledWith("Trace batch experiment read",@@ -139,10 +180,52 @@ describe("trace batch queue", () => {buildId: "test-build",maxThreads: 1,maxBlockSize: 512,- batchTraceCount: 1,+ batchTraceCount: 2,+ jobId: "batch-job",+ attempt: 2,+ queryId: vi.mocked(getTraceBatchEventStream).mock.lastCall![1]!+ .queryId,+ }),+ );+ expect(log).toHaveBeenCalledWith(+ "Trace batch experiment read completed",+ expect.objectContaining({+ jobId: "batch-job",+ attempt: 2,+ queryId: vi.mocked(getTraceBatchEventStream).mock.lastCall![1]!+ .queryId,+ outcome: "failure",+ batchTraceCount: 2,+ batchProjectCount: 2,+ eventTimeSpanMs: 11_000,+ maxTraceSpanMs: 3_000,+ observationCount: 1,+ foundTraceCount: 1,+ foundProjectCount: 1,+ inputBytes: 5,+ outputBytes: 6,+ metadataBytes: 9,+ partial: true,+ durationMs: expect.any(Number),}),);- expect(recordDistribution).not.toHaveBeenCalled();+ expect(vi.mocked(recordDistribution).mock.calls).toEqual([+ ["langfuse.trace_batch.failed_read_observation_count", 1],+ ["langfuse.trace_batch.failed_read_input_bytes", 5],+ ["langfuse.trace_batch.failed_read_output_bytes", 6],+ ["langfuse.trace_batch.failed_read_metadata_bytes", 9],+ ["langfuse.trace_batch.failed_read_io_metadata_bytes", 20],+ [+ "langfuse.trace_batch.read_duration_ms",+ expect.any(Number),+ { outcome: "failure" },+ ],+ ]);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.read_attempts",+ 1,+ { outcome: "failure" },+ );} finally {env.LANGFUSE_TRACE_BATCH_MAX_THREADS =originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;@@ -153,7 +236,89 @@ describe("trace batch queue", () => {env.BUILD_ID = originalEnv.BUILD_ID;}});+ it("retains the active read count when overlapping reads succeed or fail", async () => {+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "overlapping";+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);+ const first = Promise.withResolvers<void>();+ const second = Promise.withResolvers<void>();+ const failure = new Error("stream failed");+ vi.mocked(getTraceBatchEventStream)+ .mockImplementationOnce(async function* () {+ await first.promise;+ })+ .mockImplementationOnce(async function* () {+ await second.promise;+ throw failure;+ });+ const job = {+ data: {+ id: "batch",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(),+ payload: {+ traces: [+ {+ projectId: "project",+ traceId: "trace",+ minStart: 0,+ maxStart: 1,+ revision: "r",+ },+ ],+ },+ },+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;++ const successfulRead = traceBatchQueueProcessor(job, undefined);+ const failedRead = traceBatchQueueProcessor(job, undefined);+ expect(recordGauge).toHaveBeenLastCalledWith(+ "langfuse.trace_batch.active_reads",+ 2,+ );+ // Periodic samples keep unchanged, long-running reads visible.+ recordTraceBatchActiveReads();+ first.resolve();+ await successfulRead;+ expect(recordGauge).toHaveBeenLastCalledWith(+ "langfuse.trace_batch.active_reads",+ 1,+ );+ second.resolve();+ await expect(failedRead).rejects.toBe(failure);+ for (const outcome of ["success", "failure"]) {+ expect(log).toHaveBeenCalledWith(+ "Trace batch experiment read completed",+ expect.objectContaining({+ outcome,+ batchTraceCount: 1,+ batchProjectCount: 1,+ eventTimeSpanMs: 1,+ maxTraceSpanMs: 1,+ observationCount: 0,+ foundTraceCount: 0,+ foundProjectCount: 0,+ inputBytes: 0,+ outputBytes: 0,+ metadataBytes: 0,+ partial: outcome === "failure",+ }),+ );+ }+ expect(vi.mocked(recordGauge).mock.calls).toEqual([+ ["langfuse.trace_batch.active_reads", 1],+ ["langfuse.trace_batch.active_reads", 2],+ ["langfuse.trace_batch.active_reads", 2],+ ["langfuse.trace_batch.active_reads", 1],+ ["langfuse.trace_batch.active_reads", 0],+ ]);+ expect(vi.mocked(recordIncrement).mock.calls).toEqual([+ ["langfuse.trace_batch.read_attempts", 1, { outcome: "success" }],+ ["langfuse.trace_batch.read_attempts", 1, { outcome: "failure" }],+ ]);+ });it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "successful-read";+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);const ingestionEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;const dispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";@@ -166,11 +331,11 @@ describe("trace batch queue", () => {["other", "trace-a"],["project", "trace-missing"],["absent", "trace-missing"],- ].map(([projectId, traceId]) => ({+ ].map(([projectId, traceId], index) => ({projectId,traceId,- minStart: 1_000,- maxStart: 2_000,+ minStart: 1_000 + index * 10_000,+ maxStart: 2_000 + index * 11_000,revision: "revision",})),};@@ -231,6 +396,24 @@ describe("trace batch queue", () => {expect.anything(),);expect(yieldedRows).toBe(8);+ expect(log).toHaveBeenCalledWith(+ "Trace batch experiment read completed",+ expect.objectContaining({+ outcome: "success",+ batchTraceCount: 5,+ batchProjectCount: 3,+ eventTimeSpanMs: 45_000,+ maxTraceSpanMs: 5_000,+ observationCount: 4,+ foundTraceCount: 3,+ foundProjectCount: 2,+ inputBytes: 20,+ outputBytes: 24,+ metadataBytes: 36,+ partial: false,+ durationMs: expect.any(Number),+ }),+ );for (const [name, bytes] of [["input_bytes", 20],["output_bytes", 24],diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex 834f0b4353..73592f2cac 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -1,8 +1,10 @@import { type Processor } from "bullmq";+import { randomUUID } from "node:crypto";import {getTraceBatchEventStream,logger,recordDistribution,+ recordGauge,recordIncrement,TraceBatchEventSchema,type QueueName,@@ -11,101 +13,194 @@ import {import { env } from "../env";const JOB_MAX_AGE_MS = 2 * 60 * 60_000;+let activeReads = 0;++export function recordTraceBatchActiveReads(): void {+ recordGauge("langfuse.trace_batch.active_reads", activeReads);+}export const traceBatchQueueProcessor: Processor<TQueueJobTypes[QueueName.TraceBatch]> = async (job) => {- const disabledReason = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION- ? "not_cloud"- : env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true"- ? "reads_disabled"- : undefined;- if (disabledReason) {- // BullMQ reads these options after the processor returns, then removes atomically.- job.opts.removeOnComplete = true;- recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {- reason: disabledReason,- });- return { discarded: disabledReason };- }- const event = TraceBatchEventSchema.parse(job.data);- if (Date.now() - event.timestamp.getTime() >= JOB_MAX_AGE_MS) {- job.opts.removeOnComplete = true;- recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {- reason: "expired",- });- return { discarded: "expired" };- }- const batch = event.payload;+ const startedAt = performance.now();+ let outcome: "success" | "failure" | "discard" = "failure";+ let queryId: string | undefined;+ let batchShape:+ | {+ batchTraceCount: number;+ batchProjectCount: number;+ eventTimeSpanMs: number;+ maxTraceSpanMs: number;+ }+ | undefined;const foundTraces = new Set<string>();const foundProjects = new Set<string>();let observationCount = 0;let inputBytes = 0;let outputBytes = 0;let metadataBytes = 0;+ try {+ const disabledReason = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION+ ? "not_cloud"+ : env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true"+ ? "reads_disabled"+ : undefined;+ if (disabledReason) {+ outcome = "discard";+ // BullMQ reads these options after the processor returns, then removes atomically.+ job.opts.removeOnComplete = true;+ recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {+ reason: disabledReason,+ });+ return { discarded: disabledReason };+ }+ const event = TraceBatchEventSchema.parse(job.data);+ if (Date.now() - event.timestamp.getTime() >= JOB_MAX_AGE_MS) {+ outcome = "discard";+ job.opts.removeOnComplete = true;+ recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {+ reason: "expired",+ });+ return { discarded: "expired" };+ }+ const batch = event.payload;+ const queryOptions = {+ maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,+ maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,+ experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,+ queryId: randomUUID(),+ };+ queryId = queryOptions.queryId;+ if (queryOptions.experimentId) {+ const projects = new Set<string>();+ let minStart = Infinity;+ let maxStart = -Infinity;+ let maxTraceSpanMs = 0;+ for (const trace of batch.traces) {+ projects.add(trace.projectId);+ minStart = Math.min(minStart, trace.minStart);+ maxStart = Math.max(maxStart, trace.maxStart);+ maxTraceSpanMs = Math.max(+ maxTraceSpanMs,+ trace.maxStart - trace.minStart,+ );+ }+ batchShape = {+ batchTraceCount: batch.traces.length,+ batchProjectCount: projects.size,+ eventTimeSpanMs: batch.traces.length ? maxStart - minStart : 0,+ maxTraceSpanMs,+ };+ logger.info("Trace batch experiment read", {+ ...queryOptions,+ jobId: job.id,+ attempt: job.attemptsMade + 1,+ // null means no override; the server profile's value is not known here.+ maxBlockSize: queryOptions.maxBlockSize ?? null,+ buildId: env.BUILD_ID,+ batchTraceCount: batch.traces.length,+ maxBatchSize: env.LANGFUSE_TRACE_BATCH_MAX_SIZE,+ concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,+ samplingRate: env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE,+ idleMs: env.LANGFUSE_TRACE_BATCH_IDLE_MS,+ dispatchIntervalMs: env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS,+ });+ }- const queryOptions = {- maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,- maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,- experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,- };- if (queryOptions.experimentId) {- logger.info("Trace batch experiment read", {- ...queryOptions,- // null means no override; the server profile's value is not known here.- maxBlockSize: queryOptions.maxBlockSize ?? null,- buildId: env.BUILD_ID,- batchTraceCount: batch.traces.length,- maxBatchSize: env.LANGFUSE_TRACE_BATCH_MAX_SIZE,- concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,- samplingRate: env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE,- idleMs: env.LANGFUSE_TRACE_BATCH_IDLE_MS,- dispatchIntervalMs: env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS,- });- }-- for await (const event of getTraceBatchEventStream(batch, queryOptions)) {- observationCount++;- foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));- foundProjects.add(event.project_id);- // Logical UTF-8 payload size, excluding JSON transport and compression.- inputBytes += Buffer.byteLength(event.input);- outputBytes += Buffer.byteLength(event.output);- for (const [key, value] of Object.entries(event.metadata)) {- metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);+ activeReads++;+ try {+ recordTraceBatchActiveReads();+ for await (const event of getTraceBatchEventStream(batch, queryOptions)) {+ observationCount++;+ foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));+ foundProjects.add(event.project_id);+ // Logical UTF-8 payload size, excluding JSON transport and compression.+ inputBytes += Buffer.byteLength(event.input);+ outputBytes += Buffer.byteLength(event.output);+ for (const [key, value] of Object.entries(event.metadata)) {+ metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);+ }+ }+ } catch (error) {+ // Only rows consumed before the failure; never count these as successful throughput.+ for (const [name, value] of [+ ["observation_count", observationCount],+ ["input_bytes", inputBytes],+ ["output_bytes", outputBytes],+ ["metadata_bytes", metadataBytes],+ ["io_metadata_bytes", inputBytes + outputBytes + metadataBytes],+ ] as const) {+ recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);+ }+ throw error;+ } finally {+ activeReads--;+ recordTraceBatchActiveReads();}- }- recordDistribution(- "langfuse.trace_batch.observation_count",- observationCount,- );- recordDistribution(- "langfuse.trace_batch.found_trace_count",- foundTraces.size,- );- const ioMetadataBytes = inputBytes + outputBytes + metadataBytes;- recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);- recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);- recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);- recordDistribution("langfuse.trace_batch.io_metadata_bytes", ioMetadataBytes);- recordDistribution(- "langfuse.trace_batch.found_project_count",- foundProjects.size,- );- recordDistribution(- "langfuse.trace_batch.missing_trace_count",- batch.traces.length - foundTraces.size,- );+ recordDistribution(+ "langfuse.trace_batch.observation_count",+ observationCount,+ );+ recordDistribution(+ "langfuse.trace_batch.found_trace_count",+ foundTraces.size,+ );+ const ioMetadataBytes = inputBytes + outputBytes + metadataBytes;+ recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);+ recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);+ recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);+ recordDistribution(+ "langfuse.trace_batch.io_metadata_bytes",+ ioMetadataBytes,+ );+ recordDistribution(+ "langfuse.trace_batch.found_project_count",+ foundProjects.size,+ );+ recordDistribution(+ "langfuse.trace_batch.missing_trace_count",+ batch.traces.length - foundTraces.size,+ );- // Retries can repeat this read; observation payloads never enter job results.- return {- observationCount,- traceCount: foundTraces.size,- projectCount: foundProjects.size,- inputBytes,- outputBytes,- metadataBytes,- ioMetadataBytes,- };+ // Retries can repeat this read; observation payloads never enter job results.+ outcome = "success";+ return {+ observationCount,+ traceCount: foundTraces.size,+ projectCount: foundProjects.size,+ inputBytes,+ outputBytes,+ metadataBytes,+ ioMetadataBytes,+ };+ } finally {+ const durationMs = performance.now() - startedAt;+ if (env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID) {+ logger.info("Trace batch experiment read completed", {+ experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,+ jobId: job.id,+ attempt: job.attemptsMade + 1,+ queryId,+ outcome,+ durationMs,+ ...batchShape,+ ...(queryId+ ? {+ observationCount,+ foundTraceCount: foundTraces.size,+ foundProjectCount: foundProjects.size,+ inputBytes,+ outputBytes,+ metadataBytes,+ partial: outcome !== "success",+ }+ : {}),+ });+ }+ recordIncrement("langfuse.trace_batch.read_attempts", 1, { outcome });+ recordDistribution("langfuse.trace_batch.read_duration_ms", durationMs, {+ outcome,+ });+ }};diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex cad437ebd2..381b4a16bb 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -25,6 +25,7 @@ import {monitorRunners,inAppAgentDlqRetryRunner,traceBatchDispatcher,+ traceBatchMetricsRunner,} from "../app";export const onShutdown: NodeJS.SignalsListener = async (signal) => {@@ -86,6 +87,7 @@ const runDrainAndClose = async () => {// Stop queue metrics runnerqueueMetricsRunner?.stop();+ traceBatchMetricsRunner?.stop();// Stop monitor runnersfor (const runner of monitorRunners) {