Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(trace-batching): link transcript spans to source traces (#17725)

Lysander Kieseld391ae21d9ad
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index 56fd322547..92a865880a 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -159,6 +159,45 @@ Datadog computes these across workers; do not average worker percentiles.
See [Datadog distributions](https://docs.datadoghq.com/metrics/distributions/).
Metric delivery and percentile configuration must be verified after deployment.
+### Inspect individual transcripts in Datadog
+
+Each completed trace emits a `trace-batch-transcript` child span under the batch
+processing span. Its attributes include `langfuse.project.id`, `langfuse.trace.id`,
+and `langfuse.trace.url` (a peek link using the configured product base URL).
+Under `langfuse.trace_batch`, the span records `transcript_tokens`, `transcript_characters`,
+`transcript_assembly_duration_ms`, `observation_count` (rows before observation
+deduplication), `has_transcript`, `tokenizer`, and `experiment_id`.
+No transcript content is attached, and IDs are not distribution metric tags.
+`transcript_characters` measures the complete transcript JSON's JavaScript string
+length (UTF-16 code units, including JSON syntax), or zero for a null transcript.
+It is recorded before tokenization, so remains available if token estimation fails.
+Only recording spans serialize this extra temporary copy; it is not retained
+while tokenization runs or included in the assembly-duration measurement.
+
+The span stays open until token estimation settles, so its duration includes
+tokenization and pool waits. Use the assembly-duration attribute for assembly
+performance. Missing estimates have `token_estimation:unavailable|failed` and no
+token count; null transcripts have zero tokens. The child span does not become
+active while the next trace streams, and failed streams do not emit a span for
+their partial final trace.
+
+In Datadog APM Trace Explorer, search for:
+
+```text
+env:prod-eu service:worker-cpu resource_name:trace-batch-transcript @langfuse.trace_batch.transcript_tokens:>=100000
+```
+
+Add token count as a numeric measure to sort largest first, display the project
+and trace IDs, and open `langfuse.trace.url`. The peek view shows the current
+source observations, not a saved copy of the measured transcript. Late arrivals
+and retries can produce different or repeated samples for the same trace.
+
+These spans follow existing APM ingestion sampling and retention. Configure a
+[custom retention filter](https://docs.datadoghq.com/tracing/trace_pipeline/trace_retention/)
+at 100% for the outlier query to keep matching ingested spans searchable. This
+cannot recover spans dropped before ingestion. Distribution metrics remain
+independent and cannot identify a historical sample's trace retroactively.
+
## Capacity measurements
The following metrics use the `langfuse.trace_batch` prefix. All additions are
diff --git a/worker/src/features/traceBatching/traceBatchTranscript.ts b/worker/src/features/traceBatching/traceBatchTranscript.ts
index 97c2d5a6bd..7f4ebd0196 100644
--- a/worker/src/features/traceBatching/traceBatchTranscript.ts
+++ b/worker/src/features/traceBatching/traceBatchTranscript.ts
@@ -1,10 +1,15 @@
import { type Model, type Observation } from "@langfuse/shared";
+import { SpanStatusCode } from "@opentelemetry/api";
import {
assembleTranscript,
+ buildTracesPath,
+ getProductBaseUrl,
+ getTracer,
orderObservations,
recordDistribution,
recordIncrement,
} from "@langfuse/shared/src/server";
+import { env } from "../../env";
import { tokenCountAsync } from "../tokenisation/async-usage";
// A fixed tokenizer makes payload sizes comparable across models and projects.
@@ -29,35 +34,89 @@ const TOKENIZER_MODEL: Model = {
export function recordTraceBatchTranscript(
observations: Observation[],
): Promise<void> {
- const startedAt = performance.now();
- const transcript = assembleTranscript(orderObservations(observations));
- recordDistribution(
- "langfuse.trace_batch.transcript_assembly_duration_ms",
- performance.now() - startedAt,
- { has_transcript: String(transcript !== null) },
- );
-
- if (transcript === null) {
- recordDistribution("langfuse.trace_batch.transcript_tokens", 0, {
- tokenizer: "o200k_base",
+ // Inherit the batch parent without activating this span while the next trace streams.
+ const span = getTracer("trace-batch").startSpan("trace-batch-transcript", {
+ attributes: {
+ "langfuse.project.id": observations[0]?.projectId,
+ "langfuse.trace.id": observations[0]?.traceId ?? undefined,
+ "langfuse.trace_batch.observation_count": observations.length,
+ "langfuse.trace_batch.experiment_id":
+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
+ "langfuse.trace_batch.tokenizer": "o200k_base",
+ },
+ });
+ try {
+ if (span.isRecording() && env.NEXTAUTH_URL && observations[0]?.traceId) {
+ const { projectId, traceId } = observations[0];
+ span.setAttribute(
+ "langfuse.trace.url",
+ new URL(
+ buildTracesPath({ projectId, query: { peek: traceId } }).slice(1),
+ getProductBaseUrl(),
+ ).toString(),
+ );
+ }
+ const startedAt = performance.now();
+ const transcript = assembleTranscript(orderObservations(observations));
+ const assemblyDurationMs = performance.now() - startedAt;
+ recordDistribution(
+ "langfuse.trace_batch.transcript_assembly_duration_ms",
+ assemblyDurationMs,
+ { has_transcript: String(transcript !== null) },
+ );
+ span.setAttributes({
+ "langfuse.trace_batch.transcript_assembly_duration_ms":
+ assemblyDurationMs,
+ "langfuse.trace_batch.has_transcript": transcript !== null,
});
- return Promise.resolve();
- }
+ if (span.isRecording()) {
+ // JSON string length in UTF-16 code units; do not retain the serialized copy.
+ span.setAttribute(
+ "langfuse.trace_batch.transcript_characters",
+ transcript === null ? 0 : JSON.stringify(transcript).length,
+ );
+ }
- // Returning only the tokenization promise lets the caller release observations
- // and stream the next trace while the tokenizer thread processes its copy.
- return tokenCountAsync({ model: TOKENIZER_MODEL, text: transcript })
- .then((tokens) => {
- if (tokens === undefined) {
- recordIncrement("langfuse.trace_batch.token_estimation_unavailable", 1);
- return;
- }
- recordDistribution("langfuse.trace_batch.transcript_tokens", tokens, {
+ if (transcript === null) {
+ recordDistribution("langfuse.trace_batch.transcript_tokens", 0, {
tokenizer: "o200k_base",
});
- })
- .catch(() => {
- // A missing experiment metric must not retry all reads in the batch.
- recordIncrement("langfuse.trace_batch.token_estimation_failed", 1);
+ span.setAttribute("langfuse.trace_batch.transcript_tokens", 0);
+ span.end();
+ return Promise.resolve();
+ }
+
+ // Callbacks capture only the span, releasing observations while the next trace streams.
+ return tokenCountAsync({ model: TOKENIZER_MODEL, text: transcript })
+ .then((tokens) => {
+ if (tokens === undefined) {
+ recordIncrement(
+ "langfuse.trace_batch.token_estimation_unavailable",
+ 1,
+ );
+ span.setAttribute(
+ "langfuse.trace_batch.token_estimation",
+ "unavailable",
+ );
+ return;
+ }
+ recordDistribution("langfuse.trace_batch.transcript_tokens", tokens, {
+ tokenizer: "o200k_base",
+ });
+ span.setAttribute("langfuse.trace_batch.transcript_tokens", tokens);
+ })
+ .catch(() => {
+ // A missing experiment metric must not retry all reads in the batch.
+ recordIncrement("langfuse.trace_batch.token_estimation_failed", 1);
+ span.setAttribute("langfuse.trace_batch.token_estimation", "failed");
+ })
+ .finally(() => span.end());
+ } catch (error) {
+ span.setStatus({
+ code: SpanStatusCode.ERROR,
+ message: "Transcript assembly failed",
});
+ span.end();
+ throw error;
+ }
}
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index da74b700bb..2732097f8e 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -1,6 +1,16 @@
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ afterAll,
+ afterEach,
+ beforeAll,
+ beforeEach,
+ describe,
+ expect,
+ it,
+ vi,
+} from "vitest";
import { type Job } from "bullmq";
-import { trace } from "@opentelemetry/api";
+import { context, SpanStatusCode, trace } from "@opentelemetry/api";
+import { NodeSDK, tracing } from "@opentelemetry/sdk-node";
import {
getCurrentSpan,
getTraceBatchEventStream,
@@ -42,7 +52,20 @@ const processingSpan = trace.wrapSpanContext({
spanId: "0123456789abcdef",
traceFlags: 1,
});
+const exporter = new tracing.InMemorySpanExporter();
+const sdk = new NodeSDK({
+ spanProcessors: [new tracing.SimpleSpanProcessor(exporter)],
+ autoDetectResources: false,
+ instrumentations: [],
+});
+beforeAll(() => sdk.start());
+afterAll(async () => {
+ await sdk.shutdown();
+ trace.disable();
+ context.disable();
+});
beforeEach(() => {
+ exporter.reset();
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
vi.mocked(getCurrentSpan).mockReturnValue(processingSpan);
@@ -117,10 +140,14 @@ describe("trace batch queue", () => {
},
},
} as Job<TQueueJobTypes[QueueName.TraceBatch]>;
- const processing = traceBatchQueueProcessor(job, undefined).then(
- (result) => ({ result, error: undefined }),
- (error: unknown) => ({ result: undefined, error }),
- );
+ const processing = context
+ .with(trace.setSpan(context.active(), processingSpan), () =>
+ traceBatchQueueProcessor(job, undefined),
+ )
+ .then(
+ (result) => ({ result, error: undefined }),
+ (error: unknown) => ({ result: undefined, error }),
+ );
void processing.then(() => (settled = true));
try {
// While a is being tokenized, all of b can arrive. At c's boundary,
@@ -128,6 +155,7 @@ describe("trace batch queue", () => {
await vi.waitFor(() => expect(suppliedThirdTrace).toBe(true));
expect(reachedEnd).toBe(false);
expect(tokenCountAsync).toHaveBeenCalledTimes(1);
+ expect(exporter.getFinishedSpans()).toHaveLength(0);
resolveFirst(100);
await vi.waitFor(() => expect(reachedEnd).toBe(true));
expect(tokenCountAsync).toHaveBeenCalledTimes(2);
@@ -137,6 +165,28 @@ describe("trace batch queue", () => {
rejectSecond(new Error("token count timed out"));
const outcome = await processing;
expect(outcome.error).toBe(streamFails ? failure : undefined);
+ const spans = exporter.getFinishedSpans();
+ expect(spans).toHaveLength(streamFails ? 2 : 3);
+ expect(spans[0].attributes).toMatchObject({
+ "langfuse.trace.id": "a",
+ "langfuse.trace_batch.transcript_tokens": 100,
+ });
+ expect(spans[1].attributes).toMatchObject({
+ "langfuse.trace.id": "b",
+ "langfuse.trace_batch.token_estimation": "failed",
+ });
+ expect(spans[1].status.code).toBe(SpanStatusCode.UNSET);
+ expect(spans[1].attributes).not.toHaveProperty(
+ "langfuse.trace_batch.transcript_tokens",
+ );
+ expect(
+ spans[1].attributes["langfuse.trace_batch.transcript_characters"],
+ ).toBeGreaterThan(0);
+ for (const span of spans) {
+ expect(span.parentSpanContext?.spanId).toBe(
+ processingSpan.spanContext().spanId,
+ );
+ }
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.token_estimation_failed",
1,
@@ -241,6 +291,28 @@ describe("trace batch queue", () => {
{ has_transcript: hasTranscript },
);
}
+ const spans = exporter.getFinishedSpans();
+ expect(spans).toHaveLength(2);
+ for (const [index, projectId] of ["a", "b"].entries()) {
+ const span = spans[index];
+ expect(span.name).toBe("trace-batch-transcript");
+ expect(span.attributes).toMatchObject({
+ "langfuse.project.id": projectId,
+ "langfuse.trace.id": "shared-trace",
+ "langfuse.trace_batch.observation_count": 2,
+ "langfuse.trace_batch.has_transcript": index === 0,
+ "langfuse.trace_batch.transcript_tokens":
+ index === 0 ? tokenCount(estimates[0][0]) : 0,
+ "langfuse.trace_batch.transcript_characters":
+ index === 0 ? serializedTranscript.length : 0,
+ "langfuse.trace_batch.transcript_assembly_duration_ms":
+ expect.any(Number),
+ });
+ const url = new URL(String(span.attributes["langfuse.trace.url"]));
+ expect(url.pathname).toBe(`/project/${projectId}/traces`);
+ expect(url.searchParams.get("peek")).toBe("shared-trace");
+ expect(JSON.stringify(span.attributes)).not.toContain("first question");
+ }
});
it("discards self-hosted jobs before parsing even when reads are enabled", async () => {
env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "discarded-read";