🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(trace-batching): link transcript spans to source traces (#17725)
Lysander Kieseld391ae21d9ad
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 56fd322547..92a865880a 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -159,6 +159,45 @@ Datadog computes these across workers; do not average worker percentiles.See [Datadog distributions](https://docs.datadoghq.com/metrics/distributions/).Metric delivery and percentile configuration must be verified after deployment.+### Inspect individual transcripts in Datadog++Each completed trace emits a `trace-batch-transcript` child span under the batch+processing span. Its attributes include `langfuse.project.id`, `langfuse.trace.id`,+and `langfuse.trace.url` (a peek link using the configured product base URL).+Under `langfuse.trace_batch`, the span records `transcript_tokens`, `transcript_characters`,+`transcript_assembly_duration_ms`, `observation_count` (rows before observation+deduplication), `has_transcript`, `tokenizer`, and `experiment_id`.+No transcript content is attached, and IDs are not distribution metric tags.+`transcript_characters` measures the complete transcript JSON's JavaScript string+length (UTF-16 code units, including JSON syntax), or zero for a null transcript.+It is recorded before tokenization, so remains available if token estimation fails.+Only recording spans serialize this extra temporary copy; it is not retained+while tokenization runs or included in the assembly-duration measurement.++The span stays open until token estimation settles, so its duration includes+tokenization and pool waits. Use the assembly-duration attribute for assembly+performance. Missing estimates have `token_estimation:unavailable|failed` and no+token count; null transcripts have zero tokens. The child span does not become+active while the next trace streams, and failed streams do not emit a span for+their partial final trace.++In Datadog APM Trace Explorer, search for:++```text+env:prod-eu service:worker-cpu resource_name:trace-batch-transcript @langfuse.trace_batch.transcript_tokens:>=100000+```++Add token count as a numeric measure to sort largest first, display the project+and trace IDs, and open `langfuse.trace.url`. The peek view shows the current+source observations, not a saved copy of the measured transcript. Late arrivals+and retries can produce different or repeated samples for the same trace.++These spans follow existing APM ingestion sampling and retention. Configure a+[custom retention filter](https://docs.datadoghq.com/tracing/trace_pipeline/trace_retention/)+at 100% for the outlier query to keep matching ingested spans searchable. This+cannot recover spans dropped before ingestion. Distribution metrics remain+independent and cannot identify a historical sample's trace retroactively.+## Capacity measurementsThe following metrics use the `langfuse.trace_batch` prefix. All additions arediff --git a/worker/src/features/traceBatching/traceBatchTranscript.ts b/worker/src/features/traceBatching/traceBatchTranscript.tsindex 97c2d5a6bd..7f4ebd0196 100644--- a/worker/src/features/traceBatching/traceBatchTranscript.ts+++ b/worker/src/features/traceBatching/traceBatchTranscript.ts@@ -1,10 +1,15 @@import { type Model, type Observation } from "@langfuse/shared";+import { SpanStatusCode } from "@opentelemetry/api";import {assembleTranscript,+ buildTracesPath,+ getProductBaseUrl,+ getTracer,orderObservations,recordDistribution,recordIncrement,} from "@langfuse/shared/src/server";+import { env } from "../../env";import { tokenCountAsync } from "../tokenisation/async-usage";// A fixed tokenizer makes payload sizes comparable across models and projects.@@ -29,35 +34,89 @@ const TOKENIZER_MODEL: Model = {export function recordTraceBatchTranscript(observations: Observation[],): Promise<void> {- const startedAt = performance.now();- const transcript = assembleTranscript(orderObservations(observations));- recordDistribution(- "langfuse.trace_batch.transcript_assembly_duration_ms",- performance.now() - startedAt,- { has_transcript: String(transcript !== null) },- );-- if (transcript === null) {- recordDistribution("langfuse.trace_batch.transcript_tokens", 0, {- tokenizer: "o200k_base",+ // Inherit the batch parent without activating this span while the next trace streams.+ const span = getTracer("trace-batch").startSpan("trace-batch-transcript", {+ attributes: {+ "langfuse.project.id": observations[0]?.projectId,+ "langfuse.trace.id": observations[0]?.traceId ?? undefined,+ "langfuse.trace_batch.observation_count": observations.length,+ "langfuse.trace_batch.experiment_id":+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,+ "langfuse.trace_batch.tokenizer": "o200k_base",+ },+ });+ try {+ if (span.isRecording() && env.NEXTAUTH_URL && observations[0]?.traceId) {+ const { projectId, traceId } = observations[0];+ span.setAttribute(+ "langfuse.trace.url",+ new URL(+ buildTracesPath({ projectId, query: { peek: traceId } }).slice(1),+ getProductBaseUrl(),+ ).toString(),+ );+ }+ const startedAt = performance.now();+ const transcript = assembleTranscript(orderObservations(observations));+ const assemblyDurationMs = performance.now() - startedAt;+ recordDistribution(+ "langfuse.trace_batch.transcript_assembly_duration_ms",+ assemblyDurationMs,+ { has_transcript: String(transcript !== null) },+ );+ span.setAttributes({+ "langfuse.trace_batch.transcript_assembly_duration_ms":+ assemblyDurationMs,+ "langfuse.trace_batch.has_transcript": transcript !== null,});- return Promise.resolve();- }+ if (span.isRecording()) {+ // JSON string length in UTF-16 code units; do not retain the serialized copy.+ span.setAttribute(+ "langfuse.trace_batch.transcript_characters",+ transcript === null ? 0 : JSON.stringify(transcript).length,+ );+ }- // Returning only the tokenization promise lets the caller release observations- // and stream the next trace while the tokenizer thread processes its copy.- return tokenCountAsync({ model: TOKENIZER_MODEL, text: transcript })- .then((tokens) => {- if (tokens === undefined) {- recordIncrement("langfuse.trace_batch.token_estimation_unavailable", 1);- return;- }- recordDistribution("langfuse.trace_batch.transcript_tokens", tokens, {+ if (transcript === null) {+ recordDistribution("langfuse.trace_batch.transcript_tokens", 0, {tokenizer: "o200k_base",});- })- .catch(() => {- // A missing experiment metric must not retry all reads in the batch.- recordIncrement("langfuse.trace_batch.token_estimation_failed", 1);+ span.setAttribute("langfuse.trace_batch.transcript_tokens", 0);+ span.end();+ return Promise.resolve();+ }++ // Callbacks capture only the span, releasing observations while the next trace streams.+ return tokenCountAsync({ model: TOKENIZER_MODEL, text: transcript })+ .then((tokens) => {+ if (tokens === undefined) {+ recordIncrement(+ "langfuse.trace_batch.token_estimation_unavailable",+ 1,+ );+ span.setAttribute(+ "langfuse.trace_batch.token_estimation",+ "unavailable",+ );+ return;+ }+ recordDistribution("langfuse.trace_batch.transcript_tokens", tokens, {+ tokenizer: "o200k_base",+ });+ span.setAttribute("langfuse.trace_batch.transcript_tokens", tokens);+ })+ .catch(() => {+ // A missing experiment metric must not retry all reads in the batch.+ recordIncrement("langfuse.trace_batch.token_estimation_failed", 1);+ span.setAttribute("langfuse.trace_batch.token_estimation", "failed");+ })+ .finally(() => span.end());+ } catch (error) {+ span.setStatus({+ code: SpanStatusCode.ERROR,+ message: "Transcript assembly failed",});+ span.end();+ throw error;+ }}diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex da74b700bb..2732097f8e 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -1,6 +1,16 @@-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ afterAll,+ afterEach,+ beforeAll,+ beforeEach,+ describe,+ expect,+ it,+ vi,+} from "vitest";import { type Job } from "bullmq";-import { trace } from "@opentelemetry/api";+import { context, SpanStatusCode, trace } from "@opentelemetry/api";+import { NodeSDK, tracing } from "@opentelemetry/sdk-node";import {getCurrentSpan,getTraceBatchEventStream,@@ -42,7 +52,20 @@ const processingSpan = trace.wrapSpanContext({spanId: "0123456789abcdef",traceFlags: 1,});+const exporter = new tracing.InMemorySpanExporter();+const sdk = new NodeSDK({+ spanProcessors: [new tracing.SimpleSpanProcessor(exporter)],+ autoDetectResources: false,+ instrumentations: [],+});+beforeAll(() => sdk.start());+afterAll(async () => {+ await sdk.shutdown();+ trace.disable();+ context.disable();+});beforeEach(() => {+ exporter.reset();env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";vi.mocked(getCurrentSpan).mockReturnValue(processingSpan);@@ -117,10 +140,14 @@ describe("trace batch queue", () => {},},} as Job<TQueueJobTypes[QueueName.TraceBatch]>;- const processing = traceBatchQueueProcessor(job, undefined).then(- (result) => ({ result, error: undefined }),- (error: unknown) => ({ result: undefined, error }),- );+ const processing = context+ .with(trace.setSpan(context.active(), processingSpan), () =>+ traceBatchQueueProcessor(job, undefined),+ )+ .then(+ (result) => ({ result, error: undefined }),+ (error: unknown) => ({ result: undefined, error }),+ );void processing.then(() => (settled = true));try {// While a is being tokenized, all of b can arrive. At c's boundary,@@ -128,6 +155,7 @@ describe("trace batch queue", () => {await vi.waitFor(() => expect(suppliedThirdTrace).toBe(true));expect(reachedEnd).toBe(false);expect(tokenCountAsync).toHaveBeenCalledTimes(1);+ expect(exporter.getFinishedSpans()).toHaveLength(0);resolveFirst(100);await vi.waitFor(() => expect(reachedEnd).toBe(true));expect(tokenCountAsync).toHaveBeenCalledTimes(2);@@ -137,6 +165,28 @@ describe("trace batch queue", () => {rejectSecond(new Error("token count timed out"));const outcome = await processing;expect(outcome.error).toBe(streamFails ? failure : undefined);+ const spans = exporter.getFinishedSpans();+ expect(spans).toHaveLength(streamFails ? 2 : 3);+ expect(spans[0].attributes).toMatchObject({+ "langfuse.trace.id": "a",+ "langfuse.trace_batch.transcript_tokens": 100,+ });+ expect(spans[1].attributes).toMatchObject({+ "langfuse.trace.id": "b",+ "langfuse.trace_batch.token_estimation": "failed",+ });+ expect(spans[1].status.code).toBe(SpanStatusCode.UNSET);+ expect(spans[1].attributes).not.toHaveProperty(+ "langfuse.trace_batch.transcript_tokens",+ );+ expect(+ spans[1].attributes["langfuse.trace_batch.transcript_characters"],+ ).toBeGreaterThan(0);+ for (const span of spans) {+ expect(span.parentSpanContext?.spanId).toBe(+ processingSpan.spanContext().spanId,+ );+ }expect(recordIncrement).toHaveBeenCalledWith("langfuse.trace_batch.token_estimation_failed",1,@@ -241,6 +291,28 @@ describe("trace batch queue", () => {{ has_transcript: hasTranscript },);}+ const spans = exporter.getFinishedSpans();+ expect(spans).toHaveLength(2);+ for (const [index, projectId] of ["a", "b"].entries()) {+ const span = spans[index];+ expect(span.name).toBe("trace-batch-transcript");+ expect(span.attributes).toMatchObject({+ "langfuse.project.id": projectId,+ "langfuse.trace.id": "shared-trace",+ "langfuse.trace_batch.observation_count": 2,+ "langfuse.trace_batch.has_transcript": index === 0,+ "langfuse.trace_batch.transcript_tokens":+ index === 0 ? tokenCount(estimates[0][0]) : 0,+ "langfuse.trace_batch.transcript_characters":+ index === 0 ? serializedTranscript.length : 0,+ "langfuse.trace_batch.transcript_assembly_duration_ms":+ expect.any(Number),+ });+ const url = new URL(String(span.attributes["langfuse.trace.url"]));+ expect(url.pathname).toBe(`/project/${projectId}/traces`);+ expect(url.searchParams.get("peek")).toBe("shared-trace");+ expect(JSON.stringify(span.attributes)).not.toContain("first question");+ }});it("discards self-hosted jobs before parsing even when reads are enabled", async () => {env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "discarded-read";