Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(trace-batching): add capacity experiment telemetry (#17614)

Lysander Kiesel6e9f8ebdff15
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index 2a8aea8f41..bbab71cfc3 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -277,7 +277,7 @@ export async function upsertClickhouse<
}
export async function* queryClickhouseStream<T>(
- opts: ClickhouseQueryOpts,
+ opts: ClickhouseQueryOpts & { queryId?: string },
): AsyncGenerator<T> {
if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);
const normalizedTags = normalizeClickHouseQueryTags(opts.tags);
@@ -288,7 +288,7 @@ export async function* queryClickhouseStream<T>(
// Client-generated so failures before/without a response still carry a
// query_id on errors and spans; system.query_log stays pollable by id.
- const queryId = randomUUID();
+ const queryId = opts.queryId ?? randomUUID();
try {
setSpanQueryAttributes(span, opts.query);
diff --git a/packages/shared/src/server/repositories/trace-batch.test.ts b/packages/shared/src/server/repositories/trace-batch.test.ts
index db8cd0b39a..619684adea 100644
--- a/packages/shared/src/server/repositories/trace-batch.test.ts
+++ b/packages/shared/src/server/repositories/trace-batch.test.ts
@@ -82,13 +82,19 @@ describe("trace batch query controls", () => {
requestedSecondRow = false;
const stream = getTraceBatchEventStream(
{ traces: members },
- { maxThreads: 1, maxBlockSize, experimentId: "arm-b" },
+ {
+ maxThreads: 1,
+ maxBlockSize,
+ experimentId: "arm-b",
+ queryId: "read-attempt",
+ },
);
expect(await stream.next()).toEqual({ value: row, done: false });
expect(requestedSecondRow).toBe(false);
const sent = vi.mocked(queryClickhouseStream).mock.lastCall![0];
expect(sent).toEqual({
...baseline,
+ queryId: "read-attempt",
tags: { ...baseline.tags, experimentId: "arm-b" },
clickhouseSettings: {
...baseline.clickhouseSettings,
diff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.ts
index 28c0b862f2..56e00e6b7d 100644
--- a/packages/shared/src/server/repositories/trace-batch.ts
+++ b/packages/shared/src/server/repositories/trace-batch.ts
@@ -196,6 +196,7 @@ export async function* getTraceBatchEventStream(
maxThreads?: number;
maxBlockSize?: number;
experimentId?: string;
+ queryId?: string;
} = {},
): AsyncGenerator<TraceBatchEventRow> {
if (props.traces.length === 0) return;
@@ -203,6 +204,7 @@ export async function* getTraceBatchEventStream(
const { query, params, projectIds } = buildTraceBatchEventQuery(props);
yield* queryClickhouseStream<TraceBatchEventRow>({
query,
+ queryId: options.queryId,
params,
useMultipartParamsAuto: true,
tags: {
diff --git a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
index 9ff6c74a2c..c71b0ef072 100644
--- a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
+++ b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
@@ -7,6 +7,7 @@ import {
ClickHouseResourceError,
} from "@langfuse/shared/src/server";
import { fail } from "assert";
+import { randomUUID } from "node:crypto";
const QUERY_ID_PATTERN =
/\[query_id: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\]/;
@@ -124,8 +125,11 @@ describe("ClickHouse Resource Error Handling", () => {
});
it("should stream successful queries", async () => {
+ const queryId = randomUUID();
const generator = queryClickhouseStream({
- query: "SELECT number FROM system.numbers LIMIT 3",
+ query:
+ "SELECT number, queryID() AS query_id FROM system.numbers LIMIT 3",
+ queryId,
});
const results = [];
@@ -137,6 +141,7 @@ describe("ClickHouse Resource Error Handling", () => {
expect(Array.isArray(results)).toBe(true);
expect(results.length).toBe(3);
expect(results[0]).toHaveProperty("number");
+ expect(results[0]).toHaveProperty("query_id", queryId);
});
});
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index c0310a725f..99e2327963 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -22,6 +22,9 @@
independently default-off and cloud-gated. Do not expose these PoC controls in
local or production env templates. Reader query controls are independent of
locality selection; logs must preserve separate input/output/metadata metrics.
+ `src/features/traceBatching/TraceBatchMetricsRunner.ts` collects bounded queue
+ and Redis snapshots independently of dispatch/consumption when either role is
+ enabled. Global snapshot gauges must not be summed across worker reporters.
- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.
- Service layer: `src/services/*`
- Rust addon (`@langfuse/native`): telemetry init and the startup hello call live
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
index 6536c90455..ef9e37e4dd 100644
--- a/worker/src/__tests__/traceBatching.test.ts
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -18,6 +18,7 @@ import {
QueueJobs,
QueueName,
recordDistribution,
+ recordGauge,
recordIncrement,
redis,
TraceBatchEventSchema,
@@ -26,6 +27,7 @@ import {
} from "@langfuse/shared/src/server";
import * as shared from "@langfuse/shared/src/server";
import { env } from "../env";
+import { TraceBatchMetricsRunner } from "../features/traceBatching/TraceBatchMetricsRunner";
import {
prepareLocalityPartials,
selectTraceBatches,
@@ -46,6 +48,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
keyPrefix: `trace-batch-test-${randomUUID()}:`,
}),
recordDistribution: vi.fn(),
+ recordGauge: vi.fn(),
recordIncrement: vi.fn(),
getS3EventStorageClient: vi.fn(),
};
@@ -415,6 +418,8 @@ describe("trace micro-batch scheduling with Redis", () => {
const originalEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
+ const originalConsumerEnabled =
+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED;
const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
const originalSamplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;
const originalStrategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;
@@ -456,6 +461,7 @@ describe("trace micro-batch scheduling with Redis", () => {
beforeEach(async () => {
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "true";
+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED = "false";
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;
@@ -480,6 +486,7 @@ describe("trace micro-batch scheduling with Redis", () => {
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = originalEnabled;
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;
+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED = originalConsumerEnabled;
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = originalSamplingRate;
env.LANGFUSE_TRACE_BATCH_STRATEGY = originalStrategy;
@@ -493,6 +500,101 @@ describe("trace micro-batch scheduling with Redis", () => {
});
afterAll(() => client().disconnect());
+ it("samples prefixed map memory and paused/delayed queue work, then reports drained zeros", async () => {
+ const now = Date.now();
+ const clock = vi.spyOn(Date, "now").mockReturnValue(now);
+ await trackTraceBatchActivity("project", [event("trace")]);
+ const data = {
+ id: randomUUID(),
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(now),
+ payload: { traces: [] },
+ };
+ const waiting = await queue.add(QueueJobs.TraceBatch, data, {
+ timestamp: now - 5_000,
+ });
+ await queue.add(QueueJobs.TraceBatch, data, { timestamp: now - 1_000 });
+ await queue.add(QueueJobs.TraceBatch, data, { delay: 120_000 });
+ await queue.pause();
+ // Age collection must not fetch or parse the batch payload.
+ await connection.hset(queue.toKey(waiting.id!), "data", "invalid JSON");
+ const metrics = new TraceBatchMetricsRunner();
+ const evaluate = vi.spyOn(client(), "eval");
+ await metrics["execute"]();
+
+ for (const key of ["due", "state"]) {
+ expect(recordGauge).toHaveBeenCalledWith(
+ "langfuse.trace_batch.redis_key_entries",
+ 1,
+ { key, unit: "records" },
+ );
+ const sample = vi
+ .mocked(recordGauge)
+ .mock.calls.find(
+ ([name, , tags]) =>
+ name === "langfuse.trace_batch.redis_key_bytes" &&
+ tags?.key === key,
+ );
+ expect(Number(sample?.[1])).toBeGreaterThan(0);
+ }
+ for (const type of ["waiting", "delayed"]) {
+ expect(recordGauge).toHaveBeenCalledWith(
+ "langfuse.trace_batch.queue_depth",
+ type === "waiting" ? 2 : 1,
+ { type, unit: "records" },
+ );
+ }
+ expect(recordGauge).toHaveBeenCalledWith(
+ "langfuse.trace_batch.queue_waiting_head_age_ms",
+ 5_000,
+ { unit: "milliseconds" },
+ );
+
+ clock.mockReturnValue(now + 30_000);
+ await metrics["execute"]();
+ expect(evaluate).toHaveBeenCalledTimes(1);
+ await client().del(dueKey, stateKey);
+ await queue.drain(true);
+ vi.mocked(recordGauge).mockClear();
+ clock.mockReturnValue(now + 60_000);
+ await metrics["execute"]();
+ expect(evaluate).toHaveBeenCalledTimes(2);
+ expect(recordGauge).toHaveBeenCalledWith(
+ "langfuse.trace_batch.active_reads",
+ 0,
+ );
+ expect(vi.mocked(recordGauge).mock.calls).toHaveLength(10);
+ expect(
+ vi.mocked(recordGauge).mock.calls.every(([, value]) => value === 0),
+ ).toBe(true);
+ });
+
+ it("awaits failed telemetry collections while preserving independent queue measurements", async () => {
+ const metrics = new TraceBatchMetricsRunner();
+ const pendingMemory = Promise.withResolvers<number[]>();
+ vi.spyOn(client(), "eval").mockImplementationOnce(
+ () => pendingMemory.promise,
+ );
+ vi.spyOn(queue, "getJobCounts").mockRejectedValueOnce(
+ new Error("queue unavailable"),
+ );
+ let completed = false;
+ const collection = metrics["execute"]().then(() => {
+ completed = true;
+ });
+ await vi.waitFor(() =>
+ expect(recordGauge).toHaveBeenCalledWith(
+ "langfuse.trace_batch.queue_waiting_head_age_ms",
+ 0,
+ { unit: "milliseconds" },
+ ),
+ );
+ expect(completed).toBe(false);
+ pendingMemory.reject(new Error("memory unavailable"));
+ await collection;
+ expect(vi.mocked(recordGauge).mock.calls).toHaveLength(2);
+ });
+
it("dispatches with locality selection and records bounded selector measurements", async () => {
env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";
const minute = 60_000;
@@ -563,6 +665,11 @@ describe("trace micro-batch scheduling with Redis", () => {
const lock = vi.spyOn(client(), "set");
await trackTraceBatchActivity("project", [event("trace")]);
await runner()["execute"]();
+ const metrics = new TraceBatchMetricsRunner();
+ await metrics["execute"]();
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+ env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "false";
+ await metrics["execute"]();
expect(evaluate).not.toHaveBeenCalled();
expect(lock).not.toHaveBeenCalled();
expect(await client().exists(dueKey, stateKey)).toBe(0);
@@ -801,8 +908,8 @@ describe("trace micro-batch scheduling with Redis", () => {
"project",
traceIds.flatMap((id) => [
event(id, 100),
- event(id, 200),
- event(id, 150),
+ event(id, 200, 200),
+ event(id, 150, 50),
]),
);
const selected = (await client().hkeys(stateKey)).sort();
@@ -820,10 +927,73 @@ describe("trace micro-batch scheduling with Redis", () => {
1_000 - expectedCount,
{ decision: "excluded" },
);
+ for (const [stage, count] of [
+ ["eligible", 1_000],
+ ["sampled", expectedCount],
+ ...(expectedCount > 0 ? [["recorded", expectedCount] as const] : []),
+ ] as const) {
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.event_updates",
+ count * 3,
+ { stage },
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.serialized_event_bytes",
+ count * 350,
+ { stage },
+ );
+ }
previous = selected;
}
});
+ it("counts only acknowledged update volume when tracking fails after a chunk", async () => {
+ const evaluate = client().eval.bind(client());
+ vi.spyOn(client(), "eval")
+ .mockImplementationOnce(evaluate)
+ .mockRejectedValueOnce(new Error("tracking connection failed"));
+
+ await expect(
+ trackTraceBatchActivity("project", [
+ ...Array.from({ length: 1_001 }, (_, index) =>
+ event(`trace-${index}`, 100, 7),
+ ),
+ { ...event("invalid"), startTimeISO: "not-a-date" },
+ ]),
+ ).resolves.toBeUndefined();
+
+ expect(await client().hlen(stateKey)).toBe(1_000);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.event_updates",
+ 1_001,
+ { stage: "eligible" },
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.event_updates",
+ 1_001,
+ { stage: "sampled" },
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.event_updates",
+ 1_000,
+ { stage: "recorded" },
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.serialized_event_bytes",
+ 7_000,
+ { stage: "recorded" },
+ );
+ expect(recordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.event_updates",
+ 1,
+ { stage: "recorded" },
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.tracking_errors",
+ 1,
+ );
+ });
+
it("preserves min/max across concurrent out-of-order arrivals, resets readiness, and separates tenants", async () => {
await Promise.all(
[30, 0, 20, 10].map((start) =>
diff --git a/worker/src/app.ts b/worker/src/app.ts
index f979e0e4c2..1d16cb798c 100644
--- a/worker/src/app.ts
+++ b/worker/src/app.ts
@@ -60,6 +60,7 @@ import { experimentCreateQueueProcessor } from "./queues/experimentQueue";
import { traceDeleteProcessor } from "./queues/traceDelete";
import { traceBatchQueueProcessor } from "./queues/traceBatchQueue";
import { TraceBatchDispatcher } from "./features/traceBatching/traceBatching";
+import { TraceBatchMetricsRunner } from "./features/traceBatching/TraceBatchMetricsRunner";
import { projectDeleteProcessor } from "./queues/projectDelete";
import {
postHogIntegrationProcessingProcessor,
@@ -156,6 +157,16 @@ if (
});
}
+export let traceBatchMetricsRunner: TraceBatchMetricsRunner | null = null;
+if (
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
+ (env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED === "true" ||
+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED === "true")
+) {
+ traceBatchMetricsRunner = new TraceBatchMetricsRunner();
+ traceBatchMetricsRunner.start();
+}
+
if (env.QUEUE_CONSUMER_TRACE_UPSERT_QUEUE_IS_ENABLED === "true") {
// Register workers for all trace upsert queue shards
const traceUpsertShardNames = TraceUpsertQueue.getShardNames();
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index 6eb79a6354..8c2f4cf8d5 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -102,6 +102,91 @@ Queries use response compression and chunked multipart parameters for up to
rather than accepting partial results. This does not cap observation count,
query memory or dispatcher snapshot memory.
+## Capacity measurements
+
+The following metrics use the `langfuse.trace_batch` prefix. All additions are
+inside the existing cloud experiment paths; normal ingestion with tracking
+disabled performs no new aggregation or Redis calls.
+
+| Metric | Kind / tags | Meaning |
+| --- | --- | --- |
+| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|sampled\|recorded` | Accepted, valid-start-time updates before sampling, after sampling, and in Redis-acknowledged chunks. Repeated updates count again. |
+| `read_attempts` | Counter; `outcome:success\|failure\|discard` | One outcome per processor invocation, including retries, validation failures and disabled/expired discards. |
+| `read_duration_ms` | Distribution; same outcome tags | Wall-clock processor duration, including failed and discarded attempts. |
+| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |
+| `failed_read_observation_count`, `failed_read_input_bytes`, `failed_read_output_bytes`, `failed_read_metadata_bytes`, `failed_read_io_metadata_bytes` | Distributions | Partial logical rows/bytes consumed before a stream failure. Separate from successful throughput. |
+| `queue_depth` | Gauge; `type:waiting\|active\|delayed\|failed` | Global BullMQ snapshots; waiting includes paused work. |
+| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |
+| `redis_key_bytes` | Gauge; `key:due\|state` | Estimated memory of each readiness key, including Redis overhead. |
+| `redis_key_entries` | Gauge; `key:due\|state` | `ZCARD` / `HLEN` at the memory snapshot. |
+
+The eligible/sampled counters reuse the already-computed serialized size and
+per-trace aggregates. Recorded volume is emitted after each acknowledged Lua
+chunk: earlier chunks remain counted if a later one fails. A timeout can mean
+Redis applied an update without acknowledgement, so the stage difference includes
+uncertain outcomes; it does not prove data loss. Writer acceptance still precedes
+the ClickHouse flush. No additional serialization is performed.
+
+`TraceBatchMetricsRunner` starts when either the dispatcher or consumer is
+enabled in a cloud worker, independently of the generic queue-metrics flag.
+It polls the queue every 30 seconds and map memory every 60 seconds per process,
+plus collection time. No completed jobs or progressing dispatcher are required.
+Queue-age collection reads only the next waiting/paused job IDs and their
+timestamp fields; it never fetches or parses batch payloads.
+One failed collection does not suppress the others; failed samples are not
+reported as zero. Check `langfuse.periodic_runner.completed` and
+`last_healthy_timestamp_seconds` with `runner:trace_batch_metrics` for freshness.
+
+Memory collection uses one bounded two-key Lua call with `MEMORY USAGE ...
+SAMPLES 5`, preserving the shared key prefix and Redis Cluster slot routing.
+Missing keys report zero; there is no keyspace scan or exact full-hash traversal.
+Memory sampling estimates storage, not process RSS or event payload size.
+
+Queue/map snapshots are global and can be emitted by several workers: use
+max/latest across reporters, never sum duplicates. For map totals, first
+deduplicate reporters per `key`, then add the due/state estimates. In contrast,
+`active_reads` is local to a worker; distinct live worker series may be summed.
+It is emitted on stream start/finish and on each metrics-runner cycle, including
+zero when idle, so unchanged long-running reads remain visible.
+Instantaneous gauges can miss short-lived peaks. A retried job may re-enter
+behind newer jobs, so waiting-head age is a bounded-cost backlog indicator, not
+an exact oldest-created-job measurement across the entire queue.
+
+With an experiment ID, start/completion logs include `jobId`, `attempt`,
+`queryId` and `experimentId`, with the completion outcome. The per-read UUID is
+forwarded to ClickHouse as `query_id` and remains the root correlation ID for
+the stream. Discards and validation failures have no query ID. Identifiers are
+not metric tags and logs contain no event payloads. Successful batch metrics and
+job return values retain their existing meanings.
+
+Completion logs also correlate the requested `batchTraceCount`, distinct
+`batchProjectCount`, `eventTimeSpanMs` (outer event-time span, without query
+buffering), and `maxTraceSpanMs` with the outcome and `durationMs`. These are
+batch-locality proxies, not measured ClickHouse scan costs. `observationCount`,
+`foundTraceCount`, `foundProjectCount` and separate `inputBytes`, `outputBytes`,
+`metadataBytes` describe rows consumed by that attempt. `partial: true` marks
+failed reads, including failures before any row arrives; zero then does not mean
+the requested batch was empty. Successful counts cover the completed stream,
+not unique ingested events. Discarded/unparsed jobs omit stream counters.
+Use these existing logs to compare batch shape and volume across outcomes by
+query ID; no payload contents, project-ID lists or per-batch metric tags are added.
+
+For the initial scaling curve, use deterministic trace sampling at 10%, 20%,
+50%, then 100% across the same project population. Annotate fixed UTC windows;
+pause tracking and drain old readiness/queued work between settings. Warm up at
+least 20 minutes and compare a settled hour at each stage, holding batch/query
+settings fixed. Plot actual admitted updates/bytes and returned rows/bytes against
+CPU, memory and backlog rather than assuming configured percentages equal load.
+These additions do not generate amplified reads or change batching, query limits,
+late-arrival behavior, queue payloads or retention.
+
+Pair the metrics with per-worker RSS/CPU/event-loop and network telemetry, Redis
+primary telemetry, and experiment-filtered ClickHouse query logs. Include failed
+attempts in CPU cost, count successful root results once, and verify distributed
+CPU-counter accounting before summing query parts. Query memory peaks are not
+actual concurrent replica RAM. Validate live metric delivery and units before
+raising production load.
+
## Compare reader settings
Establish a fresh control measurement after deploying this reader: both arms
diff --git a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts
new file mode 100644
index 0000000000..fd79cbc1ad
--- /dev/null
+++ b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts
@@ -0,0 +1,147 @@
+import {
+ logger,
+ recordGauge,
+ redis,
+ TraceBatchQueue,
+} from "@langfuse/shared/src/server";
+import { env } from "../../env";
+import { recordTraceBatchActiveReads } from "../../queues/traceBatchQueue";
+import { PeriodicRunner } from "../../utils/PeriodicRunner";
+
+const METRIC_PREFIX = "langfuse.trace_batch";
+const MEMORY_INTERVAL_MS = 60_000;
+// EVAL applies the shared client's key prefix and routes both keys to their
+// cluster slot. Raw MEMORY commands do not identify keys in ioredis metadata.
+const MEMORY_SCRIPT = `
+return {
+ redis.call('MEMORY', 'USAGE', KEYS[1], 'SAMPLES', 5) or 0,
+ redis.call('ZCARD', KEYS[1]),
+ redis.call('MEMORY', 'USAGE', KEYS[2], 'SAMPLES', 5) or 0,
+ redis.call('HLEN', KEYS[2])
+}
+`;
+
+export class TraceBatchMetricsRunner extends PeriodicRunner {
+ private lastMemorySampleAt: number | null = null;
+
+ constructor() {
+ super("trace_batch_metrics");
+ }
+
+ protected get name(): string {
+ return "trace-batch-metrics";
+ }
+
+ protected get defaultIntervalMs(): number {
+ return 30_000;
+ }
+
+ protected async execute(): Promise<void> {
+ if (
+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
+ (env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED !== "true" &&
+ env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED !== "true")
+ ) {
+ return;
+ }
+
+ recordTraceBatchActiveReads();
+ const collections = [
+ this.collectQueueDepth(),
+ this.collectWaitingHeadAge(),
+ ];
+ const now = Date.now();
+ if (
+ this.lastMemorySampleAt === null ||
+ now - this.lastMemorySampleAt >= MEMORY_INTERVAL_MS
+ ) {
+ this.lastMemorySampleAt = now;
+ collections.push(this.collectMemory());
+ }
+
+ // Each collection remains useful if another fails. Never replace a failed
+ // measurement with zero or leave issued Redis commands unawaited.
+ const results = await Promise.allSettled(collections);
+ for (const result of results) {
+ if (result.status === "rejected") {
+ this.markRunFailed(result.reason);
+ logger.error("Trace batch metrics collection failed", {
+ error: result.reason,
+ });
+ }
+ }
+ }
+
+ private async collectQueueDepth(): Promise<void> {
+ const queue = TraceBatchQueue.getInstance();
+ if (!queue) throw new Error("Trace batch queue is unavailable");
+ const counts = await queue.getJobCounts(
+ "waiting",
+ "paused",
+ "active",
+ "delayed",
+ "failed",
+ );
+ const depths = {
+ waiting: (counts.waiting ?? 0) + (counts.paused ?? 0),
+ active: counts.active ?? 0,
+ delayed: counts.delayed ?? 0,
+ failed: counts.failed ?? 0,
+ };
+ for (const [type, depth] of Object.entries(depths)) {
+ recordGauge(`${METRIC_PREFIX}.queue_depth`, depth, {
+ type,
+ unit: "records",
+ });
+ }
+ }
+
+ private async collectWaitingHeadAge(): Promise<void> {
+ const queue = TraceBatchQueue.getInstance();
+ if (!queue) throw new Error("Trace batch queue is unavailable");
+ // BullMQ returns the next FIFO entry in each of waiting and paused. A retry
+ // can re-enter behind newer jobs, so this is not the oldest creation time
+ // anywhere in the queue. Jobs can disappear between ID and hash reads.
+ const ids = await queue.getRanges(["wait", "paused"], 0, 0, true);
+ const client = await queue.client;
+ const timestamps = await Promise.all(
+ ids.map((id) => client.hget(queue.toKey(id), "timestamp")),
+ );
+ const now = Date.now();
+ const ages = timestamps.flatMap((timestamp) =>
+ timestamp !== null && Number.isFinite(Number(timestamp))
+ ? [now - Number(timestamp)]
+ : [],
+ );
+ recordGauge(
+ `${METRIC_PREFIX}.queue_waiting_head_age_ms`,
+ Math.max(0, ...ages),
+ { unit: "milliseconds" },
+ );
+ }
+
+ private async collectMemory(): Promise<void> {
+ if (!redis) throw new Error("Trace batch metrics require Redis");
+ const [dueBytes, dueEntries, stateBytes, stateEntries] = (await redis.eval(
+ MEMORY_SCRIPT,
+ 2,
+ "{trace-batch}:due",
+ "{trace-batch}:state",
+ )) as number[];
+ // These are global snapshots emitted by each eligible worker. Aggregate
+ // duplicate reporters with max/latest, never sum their identical gauges.
+ for (const { key, bytes, entries } of [
+ { key: "due", bytes: dueBytes, entries: dueEntries },
+ { key: "state", bytes: stateBytes, entries: stateEntries },
+ ]) {
+ recordGauge(`${METRIC_PREFIX}.redis_key_bytes`, bytes, {
+ key,
+ unit: "bytes",
+ });
+ recordGauge(`${METRIC_PREFIX}.redis_key_entries`, entries, {
+ key,
+ unit: "records",
+ });
+ }
+ }
+}
diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.ts
index 20f83dfe58..89b548bc4f 100644
--- a/worker/src/features/traceBatching/traceBatching.ts
+++ b/worker/src/features/traceBatching/traceBatching.ts
@@ -541,6 +541,27 @@ const ACKNOWLEDGE_SCRIPT = `
return removed
`;
+function recordTrackingVolume(
+ stage: "eligible" | "sampled" | "recorded",
+ entries: Iterable<
+ readonly [
+ string,
+ { eventUpdateCount: number; serializedEventBytes: number },
+ ]
+ >,
+): void {
+ let updates = 0;
+ let bytes = 0;
+ for (const [, state] of entries) {
+ updates += state.eventUpdateCount;
+ bytes += state.serializedEventBytes;
+ }
+ recordIncrement("langfuse.trace_batch.event_updates", updates, { stage });
+ recordIncrement("langfuse.trace_batch.serialized_event_bytes", bytes, {
+ stage,
+ });
+}
+
export async function trackTraceBatchActivity(
projectId: string,
events: {
@@ -579,6 +600,7 @@ export async function trackTraceBatchActivity(
if (bounds.size === 0) return;
try {
+ recordTrackingVolume("eligible", bounds);
const samplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;
// Sample by trace ID so later observations and retries keep the same decision.
// Dispatcher and consumer process admitted work without resampling.
@@ -601,6 +623,7 @@ export async function trackTraceBatchActivity(
bounds.size - entries.length,
{ decision: "excluded" },
);
+ recordTrackingVolume("sampled", entries);
if (entries.length === 0) return;
if (!redis) throw new Error("Trace batching requires Redis");
// Bounded scripts keep ingestion from monopolizing the global Redis slot.
@@ -625,6 +648,8 @@ export async function trackTraceBatchActivity(
]),
),
);
+ // Count only acknowledged chunks; a timeout can leave Redis's outcome unknown.
+ recordTrackingVolume("recorded", chunk);
recordIncrement("langfuse.trace_batch.expired_traces", expired);
recordIncrement("langfuse.trace_batch.tracked_traces", chunk.length);
}
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index 6edd7b65ec..0f42729d51 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -6,19 +6,27 @@ import {
QueueJobs,
type QueueName,
recordDistribution,
+ recordGauge,
+ recordIncrement,
type TQueueJobTypes,
} from "@langfuse/shared/src/server";
import { env } from "../../env";
-import { traceBatchQueueProcessor } from "../traceBatchQueue";
+import {
+ recordTraceBatchActiveReads,
+ traceBatchQueueProcessor,
+} from "../traceBatchQueue";
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
getTraceBatchEventStream: vi.fn(),
recordDistribution: vi.fn(),
+ recordGauge: vi.fn(),
+ recordIncrement: vi.fn(),
}));
const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+const originalExperimentId = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;
beforeEach(() => {
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
@@ -26,6 +34,7 @@ beforeEach(() => {
afterEach(() => {
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = originalExperimentId;
vi.restoreAllMocks();
vi.clearAllMocks();
});
@@ -41,7 +50,17 @@ describe("trace batch queue", () => {
discarded: "not_cloud",
});
expect(getTraceBatchEventStream).not.toHaveBeenCalled();
- expect(recordDistribution).not.toHaveBeenCalled();
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.read_attempts",
+ 1,
+ { outcome: "discard" },
+ );
+ expect(recordDistribution).toHaveBeenCalledExactlyOnceWith(
+ "langfuse.trace_batch.read_duration_ms",
+ expect.any(Number),
+ { outcome: "discard" },
+ );
+ expect(recordGauge).not.toHaveBeenCalled();
expect(job.opts.removeOnComplete).toBe(true);
});
@@ -71,7 +90,19 @@ describe("trace batch queue", () => {
});
}
expect(getTraceBatchEventStream).not.toHaveBeenCalled();
- expect(recordDistribution).not.toHaveBeenCalled();
+ expect(vi.mocked(recordDistribution).mock.calls).toEqual([
+ [
+ "langfuse.trace_batch.read_duration_ms",
+ expect.any(Number),
+ { outcome: "discard" },
+ ],
+ [
+ "langfuse.trace_batch.read_duration_ms",
+ expect.any(Number),
+ { outcome: "discard" },
+ ],
+ ]);
+ expect(recordGauge).not.toHaveBeenCalled();
expect(job.opts.removeOnComplete).toBe(true);
});
@@ -97,7 +128,7 @@ describe("trace batch queue", () => {
name: "generation",
input: "input",
output: "output",
- metadata: {},
+ metadata: { é: "界🙂" },
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
@@ -113,9 +144,18 @@ describe("trace batch queue", () => {
maxStart: 2_000,
revision: "r",
},
+ {
+ projectId: "other",
+ traceId: "trace",
+ minStart: 9_000,
+ maxStart: 12_000,
+ revision: "r",
+ },
],
};
const job = {
+ id: "batch-job",
+ attemptsMade: 1,
data: {
id: "batch",
name: QueueJobs.TraceBatch,
@@ -131,6 +171,7 @@ describe("trace batch queue", () => {
maxThreads: 1,
maxBlockSize: 512,
experimentId: "arm-b",
+ queryId: expect.any(String),
});
expect(log).toHaveBeenCalledWith(
"Trace batch experiment read",
@@ -139,10 +180,52 @@ describe("trace batch queue", () => {
buildId: "test-build",
maxThreads: 1,
maxBlockSize: 512,
- batchTraceCount: 1,
+ batchTraceCount: 2,
+ jobId: "batch-job",
+ attempt: 2,
+ queryId: vi.mocked(getTraceBatchEventStream).mock.lastCall![1]!
+ .queryId,
+ }),
+ );
+ expect(log).toHaveBeenCalledWith(
+ "Trace batch experiment read completed",
+ expect.objectContaining({
+ jobId: "batch-job",
+ attempt: 2,
+ queryId: vi.mocked(getTraceBatchEventStream).mock.lastCall![1]!
+ .queryId,
+ outcome: "failure",
+ batchTraceCount: 2,
+ batchProjectCount: 2,
+ eventTimeSpanMs: 11_000,
+ maxTraceSpanMs: 3_000,
+ observationCount: 1,
+ foundTraceCount: 1,
+ foundProjectCount: 1,
+ inputBytes: 5,
+ outputBytes: 6,
+ metadataBytes: 9,
+ partial: true,
+ durationMs: expect.any(Number),
}),
);
- expect(recordDistribution).not.toHaveBeenCalled();
+ expect(vi.mocked(recordDistribution).mock.calls).toEqual([
+ ["langfuse.trace_batch.failed_read_observation_count", 1],
+ ["langfuse.trace_batch.failed_read_input_bytes", 5],
+ ["langfuse.trace_batch.failed_read_output_bytes", 6],
+ ["langfuse.trace_batch.failed_read_metadata_bytes", 9],
+ ["langfuse.trace_batch.failed_read_io_metadata_bytes", 20],
+ [
+ "langfuse.trace_batch.read_duration_ms",
+ expect.any(Number),
+ { outcome: "failure" },
+ ],
+ ]);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.read_attempts",
+ 1,
+ { outcome: "failure" },
+ );
} finally {
env.LANGFUSE_TRACE_BATCH_MAX_THREADS =
originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;
@@ -153,7 +236,89 @@ describe("trace batch queue", () => {
env.BUILD_ID = originalEnv.BUILD_ID;
}
});
+ it("retains the active read count when overlapping reads succeed or fail", async () => {
+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "overlapping";
+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
+ const first = Promise.withResolvers<void>();
+ const second = Promise.withResolvers<void>();
+ const failure = new Error("stream failed");
+ vi.mocked(getTraceBatchEventStream)
+ .mockImplementationOnce(async function* () {
+ await first.promise;
+ })
+ .mockImplementationOnce(async function* () {
+ await second.promise;
+ throw failure;
+ });
+ const job = {
+ data: {
+ id: "batch",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(),
+ payload: {
+ traces: [
+ {
+ projectId: "project",
+ traceId: "trace",
+ minStart: 0,
+ maxStart: 1,
+ revision: "r",
+ },
+ ],
+ },
+ },
+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+
+ const successfulRead = traceBatchQueueProcessor(job, undefined);
+ const failedRead = traceBatchQueueProcessor(job, undefined);
+ expect(recordGauge).toHaveBeenLastCalledWith(
+ "langfuse.trace_batch.active_reads",
+ 2,
+ );
+ // Periodic samples keep unchanged, long-running reads visible.
+ recordTraceBatchActiveReads();
+ first.resolve();
+ await successfulRead;
+ expect(recordGauge).toHaveBeenLastCalledWith(
+ "langfuse.trace_batch.active_reads",
+ 1,
+ );
+ second.resolve();
+ await expect(failedRead).rejects.toBe(failure);
+ for (const outcome of ["success", "failure"]) {
+ expect(log).toHaveBeenCalledWith(
+ "Trace batch experiment read completed",
+ expect.objectContaining({
+ outcome,
+ batchTraceCount: 1,
+ batchProjectCount: 1,
+ eventTimeSpanMs: 1,
+ maxTraceSpanMs: 1,
+ observationCount: 0,
+ foundTraceCount: 0,
+ foundProjectCount: 0,
+ inputBytes: 0,
+ outputBytes: 0,
+ metadataBytes: 0,
+ partial: outcome === "failure",
+ }),
+ );
+ }
+ expect(vi.mocked(recordGauge).mock.calls).toEqual([
+ ["langfuse.trace_batch.active_reads", 1],
+ ["langfuse.trace_batch.active_reads", 2],
+ ["langfuse.trace_batch.active_reads", 2],
+ ["langfuse.trace_batch.active_reads", 1],
+ ["langfuse.trace_batch.active_reads", 0],
+ ]);
+ expect(vi.mocked(recordIncrement).mock.calls).toEqual([
+ ["langfuse.trace_batch.read_attempts", 1, { outcome: "success" }],
+ ["langfuse.trace_batch.read_attempts", 1, { outcome: "failure" }],
+ ]);
+ });
it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {
+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "successful-read";
+ const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
const ingestionEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;
const dispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
@@ -166,11 +331,11 @@ describe("trace batch queue", () => {
["other", "trace-a"],
["project", "trace-missing"],
["absent", "trace-missing"],
- ].map(([projectId, traceId]) => ({
+ ].map(([projectId, traceId], index) => ({
projectId,
traceId,
- minStart: 1_000,
- maxStart: 2_000,
+ minStart: 1_000 + index * 10_000,
+ maxStart: 2_000 + index * 11_000,
revision: "revision",
})),
};
@@ -231,6 +396,24 @@ describe("trace batch queue", () => {
expect.anything(),
);
expect(yieldedRows).toBe(8);
+ expect(log).toHaveBeenCalledWith(
+ "Trace batch experiment read completed",
+ expect.objectContaining({
+ outcome: "success",
+ batchTraceCount: 5,
+ batchProjectCount: 3,
+ eventTimeSpanMs: 45_000,
+ maxTraceSpanMs: 5_000,
+ observationCount: 4,
+ foundTraceCount: 3,
+ foundProjectCount: 2,
+ inputBytes: 20,
+ outputBytes: 24,
+ metadataBytes: 36,
+ partial: false,
+ durationMs: expect.any(Number),
+ }),
+ );
for (const [name, bytes] of [
["input_bytes", 20],
["output_bytes", 24],
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index 834f0b4353..73592f2cac 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -1,8 +1,10 @@
import { type Processor } from "bullmq";
+import { randomUUID } from "node:crypto";
import {
getTraceBatchEventStream,
logger,
recordDistribution,
+ recordGauge,
recordIncrement,
TraceBatchEventSchema,
type QueueName,
@@ -11,101 +13,194 @@ import {
import { env } from "../env";
const JOB_MAX_AGE_MS = 2 * 60 * 60_000;
+let activeReads = 0;
+
+export function recordTraceBatchActiveReads(): void {
+ recordGauge("langfuse.trace_batch.active_reads", activeReads);
+}
export const traceBatchQueueProcessor: Processor<
TQueueJobTypes[QueueName.TraceBatch]
> = async (job) => {
- const disabledReason = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
- ? "not_cloud"
- : env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true"
- ? "reads_disabled"
- : undefined;
- if (disabledReason) {
- // BullMQ reads these options after the processor returns, then removes atomically.
- job.opts.removeOnComplete = true;
- recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {
- reason: disabledReason,
- });
- return { discarded: disabledReason };
- }
- const event = TraceBatchEventSchema.parse(job.data);
- if (Date.now() - event.timestamp.getTime() >= JOB_MAX_AGE_MS) {
- job.opts.removeOnComplete = true;
- recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {
- reason: "expired",
- });
- return { discarded: "expired" };
- }
- const batch = event.payload;
+ const startedAt = performance.now();
+ let outcome: "success" | "failure" | "discard" = "failure";
+ let queryId: string | undefined;
+ let batchShape:
+ | {
+ batchTraceCount: number;
+ batchProjectCount: number;
+ eventTimeSpanMs: number;
+ maxTraceSpanMs: number;
+ }
+ | undefined;
const foundTraces = new Set<string>();
const foundProjects = new Set<string>();
let observationCount = 0;
let inputBytes = 0;
let outputBytes = 0;
let metadataBytes = 0;
+ try {
+ const disabledReason = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+ ? "not_cloud"
+ : env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true"
+ ? "reads_disabled"
+ : undefined;
+ if (disabledReason) {
+ outcome = "discard";
+ // BullMQ reads these options after the processor returns, then removes atomically.
+ job.opts.removeOnComplete = true;
+ recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {
+ reason: disabledReason,
+ });
+ return { discarded: disabledReason };
+ }
+ const event = TraceBatchEventSchema.parse(job.data);
+ if (Date.now() - event.timestamp.getTime() >= JOB_MAX_AGE_MS) {
+ outcome = "discard";
+ job.opts.removeOnComplete = true;
+ recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {
+ reason: "expired",
+ });
+ return { discarded: "expired" };
+ }
+ const batch = event.payload;
+ const queryOptions = {
+ maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,
+ maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,
+ experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
+ queryId: randomUUID(),
+ };
+ queryId = queryOptions.queryId;
+ if (queryOptions.experimentId) {
+ const projects = new Set<string>();
+ let minStart = Infinity;
+ let maxStart = -Infinity;
+ let maxTraceSpanMs = 0;
+ for (const trace of batch.traces) {
+ projects.add(trace.projectId);
+ minStart = Math.min(minStart, trace.minStart);
+ maxStart = Math.max(maxStart, trace.maxStart);
+ maxTraceSpanMs = Math.max(
+ maxTraceSpanMs,
+ trace.maxStart - trace.minStart,
+ );
+ }
+ batchShape = {
+ batchTraceCount: batch.traces.length,
+ batchProjectCount: projects.size,
+ eventTimeSpanMs: batch.traces.length ? maxStart - minStart : 0,
+ maxTraceSpanMs,
+ };
+ logger.info("Trace batch experiment read", {
+ ...queryOptions,
+ jobId: job.id,
+ attempt: job.attemptsMade + 1,
+ // null means no override; the server profile's value is not known here.
+ maxBlockSize: queryOptions.maxBlockSize ?? null,
+ buildId: env.BUILD_ID,
+ batchTraceCount: batch.traces.length,
+ maxBatchSize: env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
+ concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,
+ samplingRate: env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE,
+ idleMs: env.LANGFUSE_TRACE_BATCH_IDLE_MS,
+ dispatchIntervalMs: env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS,
+ });
+ }
- const queryOptions = {
- maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,
- maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,
- experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
- };
- if (queryOptions.experimentId) {
- logger.info("Trace batch experiment read", {
- ...queryOptions,
- // null means no override; the server profile's value is not known here.
- maxBlockSize: queryOptions.maxBlockSize ?? null,
- buildId: env.BUILD_ID,
- batchTraceCount: batch.traces.length,
- maxBatchSize: env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
- concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,
- samplingRate: env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE,
- idleMs: env.LANGFUSE_TRACE_BATCH_IDLE_MS,
- dispatchIntervalMs: env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS,
- });
- }
-
- for await (const event of getTraceBatchEventStream(batch, queryOptions)) {
- observationCount++;
- foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));
- foundProjects.add(event.project_id);
- // Logical UTF-8 payload size, excluding JSON transport and compression.
- inputBytes += Buffer.byteLength(event.input);
- outputBytes += Buffer.byteLength(event.output);
- for (const [key, value] of Object.entries(event.metadata)) {
- metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);
+ activeReads++;
+ try {
+ recordTraceBatchActiveReads();
+ for await (const event of getTraceBatchEventStream(batch, queryOptions)) {
+ observationCount++;
+ foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));
+ foundProjects.add(event.project_id);
+ // Logical UTF-8 payload size, excluding JSON transport and compression.
+ inputBytes += Buffer.byteLength(event.input);
+ outputBytes += Buffer.byteLength(event.output);
+ for (const [key, value] of Object.entries(event.metadata)) {
+ metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);
+ }
+ }
+ } catch (error) {
+ // Only rows consumed before the failure; never count these as successful throughput.
+ for (const [name, value] of [
+ ["observation_count", observationCount],
+ ["input_bytes", inputBytes],
+ ["output_bytes", outputBytes],
+ ["metadata_bytes", metadataBytes],
+ ["io_metadata_bytes", inputBytes + outputBytes + metadataBytes],
+ ] as const) {
+ recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);
+ }
+ throw error;
+ } finally {
+ activeReads--;
+ recordTraceBatchActiveReads();
}
- }
- recordDistribution(
- "langfuse.trace_batch.observation_count",
- observationCount,
- );
- recordDistribution(
- "langfuse.trace_batch.found_trace_count",
- foundTraces.size,
- );
- const ioMetadataBytes = inputBytes + outputBytes + metadataBytes;
- recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);
- recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);
- recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);
- recordDistribution("langfuse.trace_batch.io_metadata_bytes", ioMetadataBytes);
- recordDistribution(
- "langfuse.trace_batch.found_project_count",
- foundProjects.size,
- );
- recordDistribution(
- "langfuse.trace_batch.missing_trace_count",
- batch.traces.length - foundTraces.size,
- );
+ recordDistribution(
+ "langfuse.trace_batch.observation_count",
+ observationCount,
+ );
+ recordDistribution(
+ "langfuse.trace_batch.found_trace_count",
+ foundTraces.size,
+ );
+ const ioMetadataBytes = inputBytes + outputBytes + metadataBytes;
+ recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);
+ recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);
+ recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);
+ recordDistribution(
+ "langfuse.trace_batch.io_metadata_bytes",
+ ioMetadataBytes,
+ );
+ recordDistribution(
+ "langfuse.trace_batch.found_project_count",
+ foundProjects.size,
+ );
+ recordDistribution(
+ "langfuse.trace_batch.missing_trace_count",
+ batch.traces.length - foundTraces.size,
+ );
- // Retries can repeat this read; observation payloads never enter job results.
- return {
- observationCount,
- traceCount: foundTraces.size,
- projectCount: foundProjects.size,
- inputBytes,
- outputBytes,
- metadataBytes,
- ioMetadataBytes,
- };
+ // Retries can repeat this read; observation payloads never enter job results.
+ outcome = "success";
+ return {
+ observationCount,
+ traceCount: foundTraces.size,
+ projectCount: foundProjects.size,
+ inputBytes,
+ outputBytes,
+ metadataBytes,
+ ioMetadataBytes,
+ };
+ } finally {
+ const durationMs = performance.now() - startedAt;
+ if (env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID) {
+ logger.info("Trace batch experiment read completed", {
+ experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
+ jobId: job.id,
+ attempt: job.attemptsMade + 1,
+ queryId,
+ outcome,
+ durationMs,
+ ...batchShape,
+ ...(queryId
+ ? {
+ observationCount,
+ foundTraceCount: foundTraces.size,
+ foundProjectCount: foundProjects.size,
+ inputBytes,
+ outputBytes,
+ metadataBytes,
+ partial: outcome !== "success",
+ }
+ : {}),
+ });
+ }
+ recordIncrement("langfuse.trace_batch.read_attempts", 1, { outcome });
+ recordDistribution("langfuse.trace_batch.read_duration_ms", durationMs, {
+ outcome,
+ });
+ }
};
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index cad437ebd2..381b4a16bb 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -25,6 +25,7 @@ import {
monitorRunners,
inAppAgentDlqRetryRunner,
traceBatchDispatcher,
+ traceBatchMetricsRunner,
} from "../app";
export const onShutdown: NodeJS.SignalsListener = async (signal) => {
@@ -86,6 +87,7 @@ const runDrainAndClose = async () => {
// Stop queue metrics runner
queueMetricsRunner?.stop();
+ traceBatchMetricsRunner?.stop();
// Stop monitor runners
for (const runner of monitorRunners) {