🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(trace-batching): assemble streamed transcripts and measure token sizes (#17649)
Lysander Kiesel51327b1db347
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex 72b7a947ce..4d318b0b3b 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -32,6 +32,8 @@the worker owns experiment enablement and lifecycle. Reader options controlper-query threads/block size and experiment attribution; retain exact tenantpairs and per-trace time windows when changing parameter chunking.+ Rows must stay contiguous by project and trace ID so consumers can finish a+ trace at the next pair or successful EOF; time buckets must not precede trace ID.- Code evaluator dispatcher/error contract: `src/server/evals/codeEvalDispatcherTypes.ts`. Keep provider mappings, user-visible messages, and worker terminal-outcome classification aligned when adding an error code.- Dashboard/monitor query feature (data model + server-only builder/executor): `src/features/query/*`- Query-builder AST (server half, WIP): `src/server/query-ast/*` — the Kyselydiff --git a/packages/shared/src/server/repositories/trace-batch.ts b/packages/shared/src/server/repositories/trace-batch.tsindex 56e00e6b7d..4c291ee31e 100644--- a/packages/shared/src/server/repositories/trace-batch.ts+++ b/packages/shared/src/server/repositories/trace-batch.ts@@ -177,7 +177,13 @@ const buildTraceBatchEventQuery = (props: TraceBatchEventStreamProps) => {`(${timeGroupPredicates.join(`OR `)})`,timeGroupParams,- );+ )+ // Keep each tenant/trace contiguous across time buckets and result blocks.+ // The transcript assembler orders observations within each trace in memory.+ .orderByColumns([+ { column: "e.project_id", direction: "ASC" },+ { column: "e.trace_id", direction: "ASC" },+ ]);return {...builder.buildWithParams(),@@ -187,6 +193,8 @@ const buildTraceBatchEventQuery = (props: TraceBatchEventStreamProps) => {/*** Stream full events_full payloads into the worker without retaining a whole batch.+ * Rows are contiguous per (project_id, trace_id); a pair change or successful EOF+ * completes that trace's query window, not its lifetime of possible late arrivals.* Rows reflect the events table's current merge state, as in other event reads;* this read-only experiment does not deduplicate versions or enrich model data.*/diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex 6dcb82efce..b415d2e92c 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -191,6 +191,7 @@ describe("Clickhouse Events Repository Test", () => {let otherProjectWindowRowSeen = false;let repeatedIntervalRowCount = 0;const foundTraces = new Set<string>();+ const traceGroups: string[] = [];for await (const event of getTraceBatchEventStream({traces: [{@@ -241,7 +242,9 @@ describe("Clickhouse Events Repository Test", () => {],})) {rowCount++;- foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));+ const traceKey = JSON.stringify([event.project_id, event.trace_id]);+ foundTraces.add(traceKey);+ if (traceGroups.at(-1) !== traceKey) traceGroups.push(traceKey);if (event.span_id === companionOnlyRow.span_id) {companionOnlyRowSeen = true;}@@ -267,6 +270,9 @@ describe("Clickhouse Events Repository Test", () => {expect(companionOnlyRowSeen).toBe(false);expect(otherProjectWindowRowSeen).toBe(false);expect(repeatedIntervalRowCount).toBe(1);+ // Interleaved timestamps across minutes must not reopen a completed trace;+ // the same trace ID in another project must form its own group.+ expect(traceGroups).toEqual([...foundTraces].sort());expect(foundTraces).toEqual(new Set([JSON.stringify([batchProjectId, firstTraceId]),diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex 99e2327963..6c22c01c3b 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -25,6 +25,10 @@`src/features/traceBatching/TraceBatchMetricsRunner.ts` collects bounded queueand Redis snapshots independently of dispatch/consumption when either role isenabled. Global snapshot gauges must not be summed across worker reporters.+ `src/features/traceBatching/traceBatchTranscript.ts` measures per-trace assembly+ and token estimates. Allow one pending tokenization promise per batch while+ buffering the next trace, and drain it even on read failure. Never flush a failed+ stream's partial final trace; completion covers the query window, not future arrivals.- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.- Service layer: `src/services/*`- Rust addon (`@langfuse/native`): telemetry init and the startup hello call livediff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 69b5c4cdbf..56fd322547 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -1,6 +1,7 @@# Internal cloud trace-batch experiment-This read-only experiment measures full-event reads for idle traces. It requires+This read-only experiment measures full-event reads and transcript assembly for+idle traces. It requires`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` and explicit opt-in. All four enablementflags below default to `false`; an ordinary release needs no infrastructurechanges. These internal controls are intentionally absent from env templates.@@ -102,6 +103,62 @@ Queries use response compression and chunked multipart parameters for up torather than accepting partial results. This does not cap observation count,query memory or dispatcher snapshot memory.+## Per-trace transcripts++The reader orders only by project ID and trace ID. The worker keeps+one trace's observations, assembles it when the pair changes, and flushes the last+trace only after a successful end of stream. A stream error discards the current+partial trace; earlier completed traces have already emitted samples. Each trace+uses the shared depth-first ordering and transcript assembler. Transcripts are+ephemeral: neither logs nor job results contain their content.++Completion means all rows returned for that pair's query window, not that no late+events can arrive. Event versions follow the current ClickHouse merge state;+the shared ordering function keeps one observation per ID. Retries can repeat+per-trace samples.++These distributions use the `langfuse.trace_batch` prefix:++| Metric | Sample |+| --- | --- |+| `transcript_assembly_duration_ms` | One trace's ordering and assembly time, excluding I/O conversion, stream waits and tokenization; `has_transcript:true\|false`. |+| `transcript_tokens` | Token estimate of the complete transcript JSON (history, current turn and provenance); zero when no transcript can be assembled. |++Transcript token counts use the existing local worker-thread pool and bundled+tiktoken WASM, without a network or model API call. The `gpt-4o` configuration+selects `o200k_base`, also used by GPT-5 mini and nano+([OpenAI mapping](https://github.com/openai/tiktoken/blob/main/tiktoken/model.py)); metrics are tagged+`tokenizer:o200k_base`. These are serialized-payload estimates, not provider+billing counts or a model's full request framing.+Unknown estimates are omitted and counted in `token_estimation_unavailable`.+Rejected estimates increment `token_estimation_failed` and omit the token sample;+they do not retry the batch. Only the assembled transcript is tokenized.++The worker submits one transcript for tokenization while streaming the next+trace's observations. Before submitting another transcript it awaits the previous+promise, keeping at most one pending token estimate per batch. Success and stream+failure both drain accepted promises; a stream failure never assembles its+partial final trace. Assembly itself remains synchronous. `read_duration_ms`+includes all this processing.++Memory includes the current trace, the previous transcript being tokenized and+stream buffers, multiplied by active batch jobs. A single trace remains unbounded.+The default two-thread tokenizer pool is shared with ingestion; overlap hides+waiting time but does not remove CPU use or contention. Its existing 30-second+timeout rejects the promise without cancelling queued/running encoding, so the+pending-promise bound is not a cancellation guarantee. Watch tokenization failures+alongside `runtime.node.mem.rss`, `runtime.node.mem.heap_used`, worker+CPU and queue depth before raising batch concurrency.++ClickHouse must sort the filtered result, so compare query memory and latency against the unordered+baseline before increasing load.++Enable percentile aggregations for these distribution metrics in Datadog+Metrics Summary, then select p50, p75, p90, p95 and p99 in Metrics Explorer.+Datadog computes these across workers; do not average worker percentiles.+See [Datadog distributions](https://docs.datadoghq.com/metrics/distributions/).+Metric delivery and percentile configuration must be verified after deployment.+## Capacity measurementsThe following metrics use the `langfuse.trace_batch` prefix. All additions arediff --git a/worker/src/features/traceBatching/traceBatchTranscript.ts b/worker/src/features/traceBatching/traceBatchTranscript.tsnew file mode 100644index 0000000000..97c2d5a6bd--- /dev/null+++ b/worker/src/features/traceBatching/traceBatchTranscript.ts@@ -0,0 +1,63 @@+import { type Model, type Observation } from "@langfuse/shared";+import {+ assembleTranscript,+ orderObservations,+ recordDistribution,+ recordIncrement,+} from "@langfuse/shared/src/server";+import { tokenCountAsync } from "../tokenisation/async-usage";++// A fixed tokenizer makes payload sizes comparable across models and projects.+// gpt-4o selects o200k_base, also used by GPT-5 mini and nano.+// This model is only an in-memory tokenizer configuration, never persisted.+const TOKENIZER_MODEL: Model = {+ id: "trace-batch-token-estimate",+ modelName: "gpt-4o",+ tokenizerId: "openai",+ tokenizerConfig: { tokenizerModel: "gpt-4o" },+ createdAt: new Date(0),+ updatedAt: new Date(0),+ projectId: null,+ matchPattern: "",+ startDate: null,+ inputPrice: null,+ outputPrice: null,+ totalPrice: null,+ unit: "TOKENS",+};++export function recordTraceBatchTranscript(+ observations: Observation[],+): Promise<void> {+ const startedAt = performance.now();+ const transcript = assembleTranscript(orderObservations(observations));+ recordDistribution(+ "langfuse.trace_batch.transcript_assembly_duration_ms",+ performance.now() - startedAt,+ { has_transcript: String(transcript !== null) },+ );++ if (transcript === null) {+ recordDistribution("langfuse.trace_batch.transcript_tokens", 0, {+ tokenizer: "o200k_base",+ });+ return Promise.resolve();+ }++ // Returning only the tokenization promise lets the caller release observations+ // and stream the next trace while the tokenizer thread processes its copy.+ return tokenCountAsync({ model: TOKENIZER_MODEL, text: transcript })+ .then((tokens) => {+ if (tokens === undefined) {+ recordIncrement("langfuse.trace_batch.token_estimation_unavailable", 1);+ return;+ }+ recordDistribution("langfuse.trace_batch.transcript_tokens", tokens, {+ tokenizer: "o200k_base",+ });+ })+ .catch(() => {+ // A missing experiment metric must not retry all reads in the batch.+ recordIncrement("langfuse.trace_batch.token_estimation_failed", 1);+ });+}diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex 7fd8df8b60..da74b700bb 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -13,6 +13,8 @@ import {type TQueueJobTypes,} from "@langfuse/shared/src/server";import { env } from "../../env";+import { tokenCountAsync } from "../../features/tokenisation/async-usage";+import { tokenCount } from "../../features/tokenisation/usage";import {recordTraceBatchActiveReads,traceBatchQueueProcessor,@@ -27,6 +29,11 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({recordIncrement: vi.fn(),}));+// Exercise real tokenization without starting the compiled worker-thread pool.+vi.mock("../../features/tokenisation/async-usage", () => ({+ tokenCountAsync: vi.fn(async (params) => tokenCount(params)),+}));+const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;const originalExperimentId = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;@@ -50,6 +57,191 @@ afterEach(() => {});describe("trace batch queue", () => {+ it.each([false, true])(+ "overlaps one tokenization with streaming, bounds pending work and drains it (stream fails: %s)",+ async (streamFails) => {+ let resolveFirst!: (tokens: number) => void;+ let rejectSecond!: (error: Error) => void;+ let cleaningUp = false;+ vi.mocked(tokenCountAsync)+ .mockImplementationOnce(+ () => new Promise<number>((resolve) => (resolveFirst = resolve)),+ )+ .mockImplementationOnce(() =>+ cleaningUp+ ? Promise.resolve(0)+ : new Promise<number>((_, reject) => (rejectSecond = reject)),+ );+ let suppliedThirdTrace = false;+ let reachedEnd = false;+ let settled = false;+ const failure = new Error("stream failed");+ vi.mocked(getTraceBatchEventStream).mockImplementation(+ async function* () {+ for (const [index, traceId] of ["a", "b", "b", "c"].entries()) {+ if (traceId === "c") suppliedThirdTrace = true;+ yield {+ project_id: "project",+ trace_id: traceId,+ span_id: `span-${index}`,+ parent_span_id: null,+ start_time: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000000",+ type: traceId === "c" ? "SPAN" : "GENERATION",+ name: "generation",+ input: "hello",+ output: "world",+ metadata: {},+ tool_definitions: {},+ tool_calls: [],+ tool_call_names: [],+ };+ }+ reachedEnd = true;+ if (streamFails) throw failure;+ },+ );+ const job = {+ data: {+ id: "bounded-tokenization",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(),+ payload: {+ traces: ["a", "b", "c"].map((traceId) => ({+ projectId: "project",+ traceId,+ minStart: 0,+ maxStart: 1,+ revision: "r",+ })),+ },+ },+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ const processing = traceBatchQueueProcessor(job, undefined).then(+ (result) => ({ result, error: undefined }),+ (error: unknown) => ({ result: undefined, error }),+ );+ void processing.then(() => (settled = true));+ try {+ // While a is being tokenized, all of b can arrive. At c's boundary,+ // processing waits for a before submitting b to the tokenizer pool.+ await vi.waitFor(() => expect(suppliedThirdTrace).toBe(true));+ expect(reachedEnd).toBe(false);+ expect(tokenCountAsync).toHaveBeenCalledTimes(1);+ resolveFirst(100);+ await vi.waitFor(() => expect(reachedEnd).toBe(true));+ expect(tokenCountAsync).toHaveBeenCalledTimes(2);+ expect(settled).toBe(false);+ // An unavailable token estimate must not retry a whole batch or mask+ // the original stream failure; its promise is drained before returning.+ rejectSecond(new Error("token count timed out"));+ const outcome = await processing;+ expect(outcome.error).toBe(streamFails ? failure : undefined);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.token_estimation_failed",+ 1,+ );+ expect(+ vi+ .mocked(recordDistribution)+ .mock.calls.filter(+ ([name]) =>+ name === "langfuse.trace_batch.transcript_assembly_duration_ms",+ ),+ ).toHaveLength(streamFails ? 2 : 3);+ } finally {+ cleaningUp = true;+ resolveFirst?.(100);+ rejectSecond?.(new Error("test cleanup"));+ await processing;+ }+ },+ );+ it("assembles each tenant's trace at the boundary and EOF, tokenizing only the transcript", async () => {+ const userMessage = { role: "user", content: "first question" };+ const answer = { role: "assistant", content: "first answer" };+ const row = {+ project_id: "a",+ trace_id: "shared-trace",+ span_id: "first",+ parent_span_id: null,+ start_time: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000000",+ type: "GENERATION",+ name: "generation",+ input: JSON.stringify([userMessage]),+ output: JSON.stringify(answer),+ metadata: {},+ tool_definitions: {},+ tool_calls: [],+ tool_call_names: [],+ };+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {+ // SQL groups traces but does not order observations within a trace.+ yield {+ ...row,+ span_id: "second",+ start_time: "2026-09-11 00:01:00.000000",+ input: JSON.stringify([+ userMessage,+ answer,+ { role: "user", content: "second question" },+ ]),+ output: JSON.stringify({ role: "assistant", content: "second answer" }),+ };+ yield row;+ expect(tokenCountAsync).not.toHaveBeenCalled();+ yield { ...row, project_id: "b", type: "SPAN" };+ // The first trace is processed before requesting more stream rows.+ expect(tokenCountAsync).toHaveBeenCalledTimes(1);+ yield { ...row, project_id: "b", type: "SPAN", span_id: "second" };+ });+ const job = {+ data: {+ id: "transcripts",+ name: QueueJobs.TraceBatch,+ timestamp: new Date(),+ payload: {+ traces: ["a", "b"].map((projectId) => ({+ projectId,+ traceId: "shared-trace",+ minStart: 0,+ maxStart: 1,+ revision: "r",+ })),+ },+ },+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;+ await traceBatchQueueProcessor(job, undefined);+ const estimates = vi.mocked(tokenCountAsync).mock.calls;+ expect(estimates).toHaveLength(1);+ const serializedTranscript = JSON.stringify(estimates[0][0].text);+ for (const content of [+ "first question",+ "first answer",+ "second question",+ "second answer",+ ]) {+ expect(serializedTranscript.split(content)).toHaveLength(2);+ }+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.transcript_tokens",+ tokenCount(estimates[0][0]),+ { tokenizer: "o200k_base" },+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.transcript_tokens",+ 0,+ { tokenizer: "o200k_base" },+ );+ for (const hasTranscript of ["true", "false"]) {+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.transcript_assembly_duration_ms",+ expect.any(Number),+ { has_transcript: hasTranscript },+ );+ }+ });it("discards self-hosted jobs before parsing even when reads are enabled", async () => {env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "discarded-read";env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;@@ -196,6 +388,7 @@ describe("trace batch queue", () => {await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(failure,);+ expect(tokenCountAsync).not.toHaveBeenCalled();expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload, {maxThreads: 1,maxBlockSize: 512,diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex deb544c28d..bfce12fc30 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -1,7 +1,9 @@/* eslint-disable no-nested-ternary */import { type Processor } from "bullmq";import { randomUUID } from "node:crypto";+import { type Observation } from "@langfuse/shared";import {+ convertObservation,getCurrentSpan,getTraceBatchEventStream,logger,@@ -13,6 +15,7 @@ import {type TQueueJobTypes,} from "@langfuse/shared/src/server";import { env } from "../env";+import { recordTraceBatchTranscript } from "../features/traceBatching/traceBatchTranscript";const JOB_MAX_AGE_MS = 2 * 60 * 60_000;let activeReads = 0;@@ -128,8 +131,10 @@ export const traceBatchQueueProcessor: Processor<}activeReads++;+ let pendingTokenization: Promise<void> | undefined;try {recordTraceBatchActiveReads();+ let traceObservations: Observation[] = [];for await (const event of getTraceBatchEventStream(batch, queryOptions)) {observationCount++;foundTraces.add(JSON.stringify([event.project_id, event.trace_id]));@@ -140,6 +145,39 @@ export const traceBatchQueueProcessor: Processor<for (const [key, value] of Object.entries(event.metadata)) {metadataBytes += Buffer.byteLength(key) + Buffer.byteLength(value);}+ const previous = traceObservations[0];+ if (+ previous &&+ (previous.projectId !== event.project_id ||+ previous.traceId !== event.trace_id)+ ) {+ // Overlap tokenization with reading the next trace, but allow only+ // one pending estimate per batch so queued payloads stay bounded.+ await pendingTokenization;+ pendingTokenization = recordTraceBatchTranscript(traceObservations);+ traceObservations = [];+ }+ traceObservations.push(+ convertObservation({+ ...event,+ id: event.span_id,+ parent_observation_id: event.parent_span_id,+ // These required converter fields are not used by the transcript.+ environment: "default",+ created_at: event.event_ts,+ updated_at: event.event_ts,+ is_deleted: 0,+ provided_usage_details: {},+ provided_cost_details: {},+ usage_details: {},+ cost_details: {},+ }),+ );+ }+ // Reaching EOF completes the last trace; a failed stream must not flush it.+ if (traceObservations.length) {+ await pendingTokenization;+ pendingTokenization = recordTraceBatchTranscript(traceObservations);}} catch (error) {// Only rows consumed before the failure; never count these as successful throughput.@@ -154,8 +192,13 @@ export const traceBatchQueueProcessor: Processor<}throw error;} finally {- activeReads--;- recordTraceBatchActiveReads();+ try {+ // Drain accepted tokenization promises even when the stream fails.+ await pendingTokenization;+ } finally {+ activeReads--;+ recordTraceBatchActiveReads();+ }}recordDistribution(