Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(transcript): collect execution issues next to the assembled transcript (#18993)

Lysander Kieselebffeda716c1
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index 844b18b9c2..f7348c94ca 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -152,6 +152,9 @@
`orderObservations`, then pass the enriched result and optional
`{ maxCharacters?, onTimings? }` to `assembleTranscript`.
The optional cap bounds `JSON.stringify(result).length`; see its README.
+ `assembleTranscriptContext` takes the same arguments and adds
+ `executionIssues` (ERROR, and WARNING with a status message) next to a
+ non-null transcript; `Transcript` itself stays conversational.
`schema.ts` owns the current transcript shape; `types.ts` infers its output
types. Message/part types come from `src/utils/normalized-io/schema.ts`.
@@ -159,8 +162,8 @@
`createTraceBatchInputStorage()` returns the `TraceBatchInputStorage` interface
with `put` and `get`, sharing a project/trace/revision key in the event bucket
and one lazily initialized provider client across factory calls. Consumers
- receive the storage object. The writer takes `TraceBatchInputData` with transcript
- and required `traceContext`; session ID and trace name are nullable. The reader
+ receive the storage object. The writer takes `TraceBatchInputData` with transcript,
+ `executionIssues` (empty without a transcript) and required `traceContext`; session ID and trace name are nullable. The reader
requires and returns both. Context timestamps
stay ISO strings. The writer skips envelopes over 1,000,000 UTF-16
units; the reader validates the stored version and restores message dates, and
diff --git a/packages/shared/src/server/traceBatch/README.md b/packages/shared/src/server/traceBatch/README.md
index cb67336422..56015f76f4 100644
--- a/packages/shared/src/server/traceBatch/README.md
+++ b/packages/shared/src/server/traceBatch/README.md
@@ -2,14 +2,16 @@
`createTraceBatchInputStorage()` provides `put` and `get` for versioned inputs,
using a project/trace/revision reference and one lazily initialized provider
-client shared across factory calls. The live inputs contain a nullable transcript
-and required trace context. Session ID and trace name are nullable; context
+client shared across factory calls. The live inputs contain a nullable transcript,
+its execution issues (empty without a transcript) and required trace context. Session ID and trace name are nullable; context
timestamps stay ISO strings.
## Stored format and versioning
-`versions/v1.ts` owns the complete V1 envelope: `schemaVersion`, `transcript`, and
-`traceContext`. It freezes the transcript and normalized message schemas with
+`versions/v1.ts` owns the complete V1 envelope: `schemaVersion`, `transcript`,
+`executionIssues`, and `traceContext`. `executionIssues` is a compatible optional
+addition: objects written before it read as an empty list, and older readers
+ignore it. It freezes the transcript and normalized message schemas with
message dates as ISO strings, importing no live schema. Its compile-time JSON
constraint rejects non-JSON fields. `serialization.ts` owns the typed
`toStoredTranscriptV1` / `fromStoredTranscriptV1` adapters between the live
diff --git a/packages/shared/src/server/traceBatch/serialization.ts b/packages/shared/src/server/traceBatch/serialization.ts
index a83e2df867..690bb68666 100644
--- a/packages/shared/src/server/traceBatch/serialization.ts
+++ b/packages/shared/src/server/traceBatch/serialization.ts
@@ -1,5 +1,5 @@
-import type { Transcript } from "../transcript/types";
-import type { StoredTranscriptV1 } from "./versions/v1";
+import type { ExecutionIssue, Transcript } from "../transcript/types";
+import type { StoredExecutionIssueV1, StoredTranscriptV1 } from "./versions/v1";
/**
* Encode the live transcript as stored V1. Only message dates need conversion;
@@ -40,3 +40,21 @@ export function fromStoredTranscriptV1(value: StoredTranscriptV1): Transcript {
})),
};
}
+
+export function toStoredExecutionIssuesV1(
+ issues: ExecutionIssue[],
+): StoredExecutionIssueV1[] {
+ return issues.map((issue) => ({
+ ...issue,
+ startTime: issue.startTime.toISOString(),
+ }));
+}
+
+export function fromStoredExecutionIssuesV1(
+ issues: StoredExecutionIssueV1[],
+): ExecutionIssue[] {
+ return issues.map((issue) => ({
+ ...issue,
+ startTime: new Date(issue.startTime),
+ }));
+}
diff --git a/packages/shared/src/server/traceBatch/storage.test.ts b/packages/shared/src/server/traceBatch/storage.test.ts
index 5cb4b1ac00..130ed6f8c9 100644
--- a/packages/shared/src/server/traceBatch/storage.test.ts
+++ b/packages/shared/src/server/traceBatch/storage.test.ts
@@ -2,7 +2,7 @@
import { Readable } from "node:stream";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { StorageService } from "../services/StorageService";
-import type { Transcript } from "../transcript/types";
+import type { ExecutionIssue, Transcript } from "../transcript/types";
import storedV1 from "./fixtures/inputs-v1.json";
import { createEventUploadStorageService } from "../s3";
import { createTraceBatchInputStorage } from "./storage";
@@ -53,7 +53,11 @@ describe("trace batch input storage", () => {
expect(createEventUploadStorageService).not.toHaveBeenCalled();
await first.get(reference);
- await second.put(reference, { transcript: null, traceContext });
+ await second.put(reference, {
+ transcript: null,
+ executionIssues: [],
+ traceContext,
+ });
expect(createEventUploadStorageService).toHaveBeenCalledExactlyOnceWith(
"events",
@@ -67,6 +71,7 @@ describe("trace batch input storage", () => {
])("round trips nested trace context with %j", async (nullableFields) => {
const inputs = {
transcript: transcript(),
+ executionIssues: [executionIssue()],
traceContext: {
...traceContext,
...nullableFields,
@@ -78,6 +83,9 @@ describe("trace batch input storage", () => {
json += chunk;
expect(JSON.parse(json)).toEqual({
...storedV1,
+ executionIssues: [
+ { ...executionIssue(), startTime: "2026-10-20T10:00:00.456Z" },
+ ],
traceContext: inputs.traceContext,
});
storage.download.mockResolvedValueOnce(json);
@@ -87,13 +95,19 @@ describe("trace batch input storage", () => {
it("reads the saved v1 format and writes it at the deterministic revision key", async () => {
const value = transcript();
const key = "test/trace-batch/trace/ecb/project/trace/revision.json";
+ // The saved fixture predates execution issues, which read as empty.
expect(await inputStorage.get(reference)).toEqual({
transcript: value,
+ executionIssues: [],
traceContext,
});
expect(storage.download).toHaveBeenCalledWith(key);
- await inputStorage.put(reference, { transcript: value, traceContext });
+ await inputStorage.put(reference, {
+ transcript: value,
+ executionIssues: [],
+ traceContext,
+ });
const upload = storage.uploadFile.mock.calls[0][0];
expect(upload).toMatchObject({
fileName: key,
@@ -102,7 +116,7 @@ describe("trace batch input storage", () => {
expect(upload.data).toBeInstanceOf(Readable);
let json = "";
for await (const chunk of upload.data) json += chunk;
- expect(JSON.parse(json)).toEqual(storedV1);
+ expect(JSON.parse(json)).toEqual({ ...storedV1, executionIssues: [] });
});
it("reads an explicitly stored null transcript", async () => {
@@ -111,6 +125,7 @@ describe("trace batch input storage", () => {
);
expect(await inputStorage.get(reference)).toEqual({
transcript: null,
+ executionIssues: [],
traceContext,
});
});
@@ -122,6 +137,7 @@ describe("trace batch input storage", () => {
const overhead = JSON.stringify({
schemaVersion: 1,
transcript: value,
+ executionIssues: [],
traceContext,
}).length;
value.threads[0].currentTurn.messages[0].parts = [
@@ -130,6 +146,7 @@ describe("trace batch input storage", () => {
expect(
await inputStorage.put(reference, {
transcript: value,
+ executionIssues: [],
traceContext,
}),
).toEqual({
@@ -147,7 +164,11 @@ describe("trace batch input storage", () => {
await expect(inputStorage.get(reference)).rejects.toThrow("NoSuchKey");
storage.uploadFile.mockRejectedValueOnce(new Error("SlowDown"));
await expect(
- inputStorage.put(reference, { transcript: null, traceContext }),
+ inputStorage.put(reference, {
+ transcript: null,
+ executionIssues: [],
+ traceContext,
+ }),
).rejects.toThrow("SlowDown");
});
@@ -166,6 +187,15 @@ describe("trace batch input storage", () => {
traceContext,
}),
],
+ [
+ "invalid execution issue",
+ JSON.stringify({
+ schemaVersion: 1,
+ transcript: null,
+ executionIssues: [{ ...executionIssue(), level: "DEFAULT" }],
+ traceContext,
+ }),
+ ],
...["sessionId", "traceName", "environment", "traceTimestamp"].map(
(field) => [
`missing trace context ${field}`,
@@ -216,3 +246,15 @@ function transcript(text = "Hello 🌍"): Transcript {
],
};
}
+
+function executionIssue(): ExecutionIssue {
+ return {
+ observationId: "tool",
+ traceId: "trace",
+ type: "TOOL",
+ name: "search",
+ level: "ERROR",
+ statusMessage: "Request timed out",
+ startTime: new Date("2026-10-20T10:00:00.456Z"),
+ };
+}
diff --git a/packages/shared/src/server/traceBatch/storage.ts b/packages/shared/src/server/traceBatch/storage.ts
index af2aaef7da..08933fea35 100644
--- a/packages/shared/src/server/traceBatch/storage.ts
+++ b/packages/shared/src/server/traceBatch/storage.ts
@@ -7,9 +7,14 @@ import {
safeBlobFilenameStem,
safeBlobKeySegment,
} from "../services/safeBlobKeySegment";
-import type { Transcript } from "../transcript/types";
+import type { ExecutionIssue, Transcript } from "../transcript/types";
import { createW3CTraceId } from "../utils/traceId";
-import { toStoredTranscriptV1, fromStoredTranscriptV1 } from "./serialization";
+import {
+ fromStoredExecutionIssuesV1,
+ fromStoredTranscriptV1,
+ toStoredExecutionIssuesV1,
+ toStoredTranscriptV1,
+} from "./serialization";
import { traceBatchInputsVersionSchema } from "./storage-schema";
import {
traceBatchInputsV1Schema,
@@ -24,6 +29,8 @@ export type TraceBatchInputReference = {
export type TraceBatchInputData = {
transcript: Transcript | null;
+ /** Empty when there is no transcript. */
+ executionIssues: ExecutionIssue[];
traceContext: TraceBatchInputsV1["traceContext"];
};
@@ -59,11 +66,12 @@ let storage: StorageService | undefined;
/** Storage dependencies share one lazily initialized provider client. */
export function createTraceBatchInputStorage(): TraceBatchInputStorage {
return {
- async put(reference, { transcript, traceContext }) {
+ async put(reference, { transcript, executionIssues, traceContext }) {
const inputs: TraceBatchInputsV1 = {
schemaVersion: 1,
transcript:
transcript === null ? null : toStoredTranscriptV1(transcript),
+ executionIssues: toStoredExecutionIssuesV1(executionIssues),
traceContext,
};
const json = JSON.stringify(inputs);
@@ -95,6 +103,7 @@ export function createTraceBatchInputStorage(): TraceBatchInputStorage {
inputs.transcript === null
? null
: fromStoredTranscriptV1(inputs.transcript),
+ executionIssues: fromStoredExecutionIssuesV1(inputs.executionIssues),
traceContext: inputs.traceContext,
};
},
diff --git a/packages/shared/src/server/traceBatch/versions/v1.ts b/packages/shared/src/server/traceBatch/versions/v1.ts
index caeeea78f4..2f5570017f 100644
--- a/packages/shared/src/server/traceBatch/versions/v1.ts
+++ b/packages/shared/src/server/traceBatch/versions/v1.ts
@@ -155,10 +155,37 @@ const storedTranscriptV1Schema = z.object({
export type StoredTranscriptV1 = z.infer<typeof storedTranscriptV1Schema>;
+const storedExecutionIssueV1Schema = z.object({
+ observationId: z.string(),
+ traceId: z.string().nullable(),
+ type: z.enum([
+ "SPAN",
+ "EVENT",
+ "GENERATION",
+ "AGENT",
+ "TOOL",
+ "CHAIN",
+ "RETRIEVER",
+ "EVALUATOR",
+ "EMBEDDING",
+ "GUARDRAIL",
+ ]),
+ name: z.string().nullable(),
+ level: z.enum(["ERROR", "WARNING"]),
+ statusMessage: z.string().nullable(),
+ startTime: isoTimestampSchema,
+}) satisfies z.ZodType<JsonValue>;
+
+export type StoredExecutionIssueV1 = z.infer<
+ typeof storedExecutionIssueV1Schema
+>;
+
// Add breaking versions alongside v1; deploy readers before enabling new writers.
export const traceBatchInputsV1Schema = z.object({
schemaVersion: z.literal(1),
transcript: storedTranscriptV1Schema.nullable(),
+ /** Optional addition: objects written before it was added read as empty. */
+ executionIssues: z.array(storedExecutionIssueV1Schema).default([]),
// Mirrors the event observation domain fields without importing the domain
// schema; the writer's typed assignment keeps both compatible.
traceContext: z.object({
diff --git a/packages/shared/src/server/transcript/README.md b/packages/shared/src/server/transcript/README.md
index dce55f6acd..07e367a345 100644
--- a/packages/shared/src/server/transcript/README.md
+++ b/packages/shared/src/server/transcript/README.md
@@ -82,6 +82,21 @@ remove that second input. Such an extension should keep conversation assembly
and trace-level facts distinct, so root I/O and operation metadata do not become
duplicate conversation messages.
+## Execution issues
+
+`assembleTranscriptContext(orderedObservations, options?)` composes
+`assembleTranscript` with execution-issue collection over the same observations
+and returns `{ transcript, executionIssues }`, or `null` when there is no
+transcript. `Transcript` stays purely conversational.
+
+An execution issue is an ERROR observation, or a WARNING observation with a
+status message (`isExecutionIssue`). Issues are collected from all supplied
+observations, including ones that contribute no messages, in observation order.
+Each keeps its observation reference, type, name, level, start time and the
+first 1,000 characters of its status message, so a consumer can place it in the
+matching observation block. `maxCharacters` limits only the transcript; issues
+are not capped.
+
## Schema
`schema.ts` owns the current transcript shape and composes the message/part schemas
@@ -284,6 +299,7 @@ transcript/
β”œβ”€β”€ ordering.ts orderObservations, the trace tree walk
β”œβ”€β”€ ordering.test.ts ordering rules
β”œβ”€β”€ transcript.ts assembleTranscript
+β”œβ”€β”€ context.ts assembleTranscriptContext and execution issues
β”œβ”€β”€ limit.ts optional hard limit on serialized JSON
β”œβ”€β”€ comparison.ts messageKey
β”œβ”€β”€ comparison.test.ts focused equality regressions
diff --git a/packages/shared/src/server/transcript/context.ts b/packages/shared/src/server/transcript/context.ts
new file mode 100644
index 0000000000..65ae035d39
--- /dev/null
+++ b/packages/shared/src/server/transcript/context.ts
@@ -0,0 +1,64 @@
+import type { OrderedObservation } from "./ordering";
+import { assembleTranscript } from "./transcript";
+import type {
+ ExecutionIssue,
+ TranscriptContext,
+ TranscriptObservation,
+ TranscriptOptions,
+} from "./types";
+
+const STATUS_MESSAGE_MAX_CHARACTERS = 1_000;
+
+/** Check if an observation's level and status message make it an execution issue. */
+export function isExecutionIssue<
+ T extends Pick<TranscriptObservation, "level" | "statusMessage">,
+>(observation: T): observation is T & { level: ExecutionIssue["level"] } {
+ return (
+ observation.level === "ERROR" ||
+ (observation.level === "WARNING" && Boolean(observation.statusMessage))
+ );
+}
+
+function toExecutionIssue(
+ observation: OrderedObservation & { level: ExecutionIssue["level"] },
+): ExecutionIssue {
+ return {
+ observationId: observation.id,
+ traceId: observation.traceId,
+ type: observation.type,
+ name: observation.name ?? null,
+ level: observation.level,
+ statusMessage: observation.statusMessage
+ ? observation.statusMessage.slice(0, STATUS_MESSAGE_MAX_CHARACTERS)
+ : null,
+ startTime: observation.startTime,
+ };
+}
+
+/** Execution issues of all supplied observations, in the given order. */
+function collectExecutionIssues(
+ observations: OrderedObservation[],
+): ExecutionIssue[] {
+ const issues: ExecutionIssue[] = [];
+ for (const observation of observations)
+ if (isExecutionIssue(observation))
+ issues.push(toExecutionIssue(observation));
+ return issues;
+}
+
+/**
+ * Assemble the transcript and collect execution issues from the same ordered
+ * observations. Null when there is no transcript: issues are only reported
+ * alongside a conversation. Options apply to the transcript only.
+ */
+export function assembleTranscriptContext(
+ orderedObservations: OrderedObservation[],
+ options?: TranscriptOptions,
+): TranscriptContext | null {
+ const transcript = assembleTranscript(orderedObservations, options);
+ if (!transcript) return null;
+ return {
+ transcript,
+ executionIssues: collectExecutionIssues(orderedObservations),
+ };
+}
diff --git a/packages/shared/src/server/transcript/fixtures/fixtures.test.ts b/packages/shared/src/server/transcript/fixtures/fixtures.test.ts
index 6e33e806cc..e0123758af 100644
--- a/packages/shared/src/server/transcript/fixtures/fixtures.test.ts
+++ b/packages/shared/src/server/transcript/fixtures/fixtures.test.ts
@@ -4,6 +4,7 @@ import { createObservation } from "../../test-utils";
import { convertObservation } from "../../repositories/observations_converters";
import { orderObservations } from "../ordering";
import { assembleTranscript } from "../transcript";
+import { assembleTranscriptContext } from "../context";
import { transcriptSchema } from "../schema";
describe("transcript fixtures", () => {
@@ -36,6 +37,66 @@ describe("transcript fixtures", () => {
]);
});
+ it("collects execution issues next to the transcript, also from observations without messages", () => {
+ const failed = (
+ id: string,
+ type: "GENERATION" | "TOOL" | "SPAN",
+ level: "ERROR" | "WARNING" | "DEFAULT",
+ statusMessage: string | null,
+ ) => ({
+ ...convertObservation(
+ createObservation({
+ id,
+ trace_id: traceId,
+ type,
+ name: `${type.toLowerCase()}-${id}`,
+ level,
+ status_message: statusMessage,
+ start_time: `2026-01-01T12:00:0${id}.000Z`,
+ input: null,
+ output: null,
+ }),
+ ),
+ nestingLevel: 1,
+ });
+ const observations = [
+ generation("1", ["A"], ["B"]),
+ failed("2", "TOOL", "ERROR", "Request timed out"),
+ failed("3", "SPAN", "WARNING", null),
+ failed("4", "SPAN", "DEFAULT", "ok"),
+ failed(
+ "5",
+ "GENERATION",
+ "ERROR",
+ "Bedrock is unable to process your request",
+ ),
+ ];
+
+ expect(assembleTranscriptContext(observations)).toEqual({
+ transcript: assembleTranscript(observations),
+ executionIssues: [
+ expect.objectContaining({
+ observationId: "2",
+ type: "TOOL",
+ name: "tool-2",
+ level: "ERROR",
+ statusMessage: "Request timed out",
+ }),
+ expect.objectContaining({
+ observationId: "5",
+ type: "GENERATION",
+ level: "ERROR",
+ statusMessage: "Bedrock is unable to process your request",
+ }),
+ ],
+ });
+ expect(assembleTranscriptContext(observations.slice(1))).toBeNull();
+ expect(assembleTranscriptContext([observations[0]])).toEqual({
+ transcript: assembleTranscript([observations[0]]),
+ executionIssues: [],
+ });
+ });
+
it("preserves an unfinished generation's null end time", () => {
const observation = generation("1", ["A"], ["B"]);
observation.endTime = null;
diff --git a/packages/shared/src/server/transcript/index.ts b/packages/shared/src/server/transcript/index.ts
index 69956333bf..e383b258a6 100644
--- a/packages/shared/src/server/transcript/index.ts
+++ b/packages/shared/src/server/transcript/index.ts
@@ -1,5 +1,6 @@
export { orderObservations } from "./ordering";
export { assembleTranscript } from "./transcript";
+export { assembleTranscriptContext, isExecutionIssue } from "./context";
export {
renderTranscript,
renderTranscriptFromObservations,
diff --git a/packages/shared/src/server/transcript/schema.ts b/packages/shared/src/server/transcript/schema.ts
index a0f5babf55..5b169202dc 100644
--- a/packages/shared/src/server/transcript/schema.ts
+++ b/packages/shared/src/server/transcript/schema.ts
@@ -1,5 +1,9 @@
import { z } from "zod";
import { normalizedMessageSchema } from "../../utils/normalized-io/schema";
+import {
+ ObservationLevelDomain,
+ ObservationTypeDomain,
+} from "../../domain/observations";
export const threadMessageSchema = normalizedMessageSchema.extend({
/** Observation that first emitted this message. */
@@ -33,3 +37,15 @@ export const transcriptSchema = z.object({
/** The character limit shortened text or omitted messages. */
truncated: z.literal(true).optional(),
});
+
+/** An ERROR observation, or a WARNING observation with a status message. */
+export const executionIssueSchema = z.object({
+ observationId: z.string(),
+ traceId: z.string().nullable(),
+ type: ObservationTypeDomain,
+ name: z.string().nullable(),
+ level: ObservationLevelDomain.extract(["ERROR", "WARNING"]),
+ /** First 1,000 characters of the status message. */
+ statusMessage: z.string().nullable(),
+ startTime: z.date(),
+});
diff --git a/packages/shared/src/server/transcript/topics-renderer.ts b/packages/shared/src/server/transcript/topics-renderer.ts
index f0c670a696..c32e8ac11f 100644
--- a/packages/shared/src/server/transcript/topics-renderer.ts
+++ b/packages/shared/src/server/transcript/topics-renderer.ts
@@ -7,6 +7,7 @@ import type {
import { normalizeIO } from "../normalized-io";
import { orderObservations } from "./ordering";
import { assembleTranscript } from "./transcript";
+import { isExecutionIssue } from "./context";
import type { Transcript } from "./types";
import {
transcriptRenderConfigSchema,
@@ -449,10 +450,7 @@ export function renderTranscript(
}
if (config.errors.include) {
observations.forEach((observation, observationIndex) => {
- if (
- observation.level === "ERROR" ||
- (observation.level === "WARNING" && observation.statusMessage)
- )
+ if (isExecutionIssue(observation))
currentWithErrors.push({
kind: "error",
observation,
diff --git a/packages/shared/src/server/transcript/types.ts b/packages/shared/src/server/transcript/types.ts
index 67042b7ee6..a4558f5fc4 100644
--- a/packages/shared/src/server/transcript/types.ts
+++ b/packages/shared/src/server/transcript/types.ts
@@ -3,6 +3,7 @@ import type {
threadMessageSchema,
turnSchema,
threadSchema,
+ executionIssueSchema,
transcriptSchema,
} from "./schema";
import type { Observation } from "../../domain";
@@ -30,9 +31,18 @@ export type TranscriptObservation = Pick<
| "input"
| "output"
| "metadata"
->;
+> &
+ Partial<Pick<Observation, "level" | "statusMessage">>;
export type ThreadMessage = z.infer<typeof threadMessageSchema>;
export type Turn = z.infer<typeof turnSchema>;
export type Thread = z.infer<typeof threadSchema>;
export type Transcript = z.infer<typeof transcriptSchema>;
+export type ExecutionIssue = z.infer<typeof executionIssueSchema>;
+
+/** A transcript and the execution issues of the same observations. */
+export type TranscriptContext = {
+ transcript: Transcript;
+ /** In observation order; empty when no observation failed. */
+ executionIssues: ExecutionIssue[];
+};
diff --git a/worker/src/features/traceBatching/traceBatchTranscript.ts b/worker/src/features/traceBatching/traceBatchTranscript.ts
index bc998c5452..c504652c10 100644
--- a/worker/src/features/traceBatching/traceBatchTranscript.ts
+++ b/worker/src/features/traceBatching/traceBatchTranscript.ts
@@ -1,7 +1,7 @@
import { type Model, type Observation } from "@langfuse/shared";
import { SpanStatusCode, type Span } from "@opentelemetry/api";
import {
- assembleTranscript,
+ assembleTranscriptContext,
buildTracesPath,
getProductBaseUrl,
getTracer,
@@ -13,6 +13,7 @@ import {
topicsTranscriptConfig,
transcriptBlockTypes,
type Transcript,
+ type TranscriptContext,
} from "@langfuse/shared/src/server";
import { env } from "../../env";
import { tokenCountAsync } from "../tokenisation/async-usage";
@@ -294,7 +295,7 @@ function recordGenericRendering(
export function recordTraceBatchTranscript(
observations: Observation[],
- onTranscript?: (transcript: Transcript | null) => Promise<void>,
+ onTranscript?: (context: TranscriptContext | null) => Promise<void>,
): Promise<void> {
// Inherit the batch parent without activating this span while the next trace streams.
const span = getTracer("trace-batch").startSpan("trace-batch-transcript", {
@@ -321,11 +322,12 @@ export function recordTraceBatchTranscript(
const startedAt = performance.now();
let phaseTimings = { normalizationMs: 0, matchingMs: 0 };
const orderedObservations = orderObservations(observations);
- const transcript = assembleTranscript(orderedObservations, {
+ const context = assembleTranscriptContext(orderedObservations, {
onTimings: (timings) => {
phaseTimings = timings;
},
});
+ const transcript = context?.transcript ?? null;
const assemblyDurationMs = performance.now() - startedAt;
recordTranscriptDistribution(
"transcript_assembly_duration_ms",
@@ -466,7 +468,7 @@ export function recordTraceBatchTranscript(
})()
.finally(() => span.end())
// Topics summarizes from the same assembled transcript.
- .then(() => onTranscript?.(transcript))
+ .then(() => onTranscript?.(context))
);
} catch (error) {
span.setStatus({
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index d2da6a6c32..32cb4baad8 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -1079,6 +1079,7 @@ describe("trace batch queue", () => {
{
transcript: vi.mocked(summarizeAssembledTrace).mock.calls[0][0]
.transcript,
+ executionIssues: [],
traceContext: {
sessionId: null,
traceTimestamp: "2026-09-11T00:00:00.000Z",
@@ -1111,6 +1112,64 @@ describe("trace batch queue", () => {
},
);
+ it("stores execution issues next to the transcript", async () => {
+ env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED = "true";
+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
+ yield {
+ project_id: "project",
+ trace_id: "trace",
+ environment: "default",
+ span_id: "span",
+ parent_span_id: null,
+ start_time: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000000",
+ type: "GENERATION",
+ name: "generation",
+ input: JSON.stringify([{ role: "user", content: "Hi" }]),
+ output: JSON.stringify({ role: "assistant", content: "Hello" }),
+ metadata: {},
+ level: "WARNING",
+ status_message: "Rate limited, retried",
+ tool_definitions: {},
+ tool_calls: [],
+ tool_call_names: [],
+ };
+ });
+ const job = {
+ data: {
+ id: "issues",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(),
+ payload: {
+ traces: [
+ {
+ projectId: "project",
+ traceId: "trace",
+ minStart: 0,
+ maxStart: 1,
+ revision: "queued-revision",
+ },
+ ],
+ },
+ },
+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+
+ await traceBatchQueueProcessor(job, undefined);
+
+ expect(inputStorage.put).toHaveBeenCalledExactlyOnceWith(
+ expect.anything(),
+ expect.objectContaining({
+ executionIssues: [
+ expect.objectContaining({
+ observationId: "span",
+ level: "WARNING",
+ statusMessage: "Rate limited, retried",
+ }),
+ ],
+ }),
+ );
+ });
+
it.each([false, true])(
"runs Topics alongside capture and holds the slot until both settle (Topics rejects: %s)",
async (summaryRejects) => {
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index 43e1af6b36..bd73af87fe 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -17,7 +17,7 @@ import {
TraceBatchEventSchema,
type QueueName,
type TQueueJobTypes,
- type Transcript,
+ type TranscriptContext,
type TraceBatchInputReference,
} from "@langfuse/shared/src/server";
import {
@@ -143,7 +143,7 @@ function newestSessionId(observations: TraceBatchObservation[]) {
async function summarizeTraceBatch(
observations: TraceBatchObservation[],
- transcript: Transcript | null,
+ context: TranscriptContext | null,
topics: TopicsBatchProjects,
jobEnqueuedAt: Date,
revisions: ReadonlyMap<string, string>,
@@ -198,14 +198,18 @@ async function summarizeTraceBatch(
...traceContext,
jobEnqueuedAt,
traceName: traceContext.traceName ?? "",
- transcript,
+ transcript: context?.transcript ?? null,
models: project.models,
facets: project.facets,
}),
inputReference
? storeInputs(
inputReference,
- { transcript, traceContext },
+ {
+ transcript: context?.transcript ?? null,
+ executionIssues: context?.executionIssues ?? [],
+ traceContext,
+ },
inputStorage,
)
: Promise.resolve(),
@@ -447,11 +451,11 @@ export const traceBatchQueueProcessor: Processor<
performance.now() - enqueuedAt,
);
if (summaryFailed) return null;
- let transcript: Transcript | null = null;
+ let context: TranscriptContext | null = null;
await recordTraceBatchTranscript(observations, async (assembled) => {
- transcript = assembled;
+ context = assembled;
});
- return { transcript };
+ return { context };
});
transcribing = transcribed.catch(() => {});
const summary: Promise<void> = transcribed
@@ -462,7 +466,7 @@ export const traceBatchQueueProcessor: Processor<
topicsOutcomes.push(
await summarizeTraceBatch(
observations,
- assembled.transcript,
+ assembled.context,
topics,
event.timestamp,
revisions,