🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(web): Improve tool response handling in sessions (#18640)
Ben Bacheme7caaad6a197
diff --git a/web/.storybook/public/revenue-chart.svg b/web/.storybook/public/revenue-chart.svgnew file mode 100644index 0000000000..ff7c585aee--- /dev/null+++ b/web/.storybook/public/revenue-chart.svg@@ -0,0 +1,17 @@+<svg xmlns="http://www.w3.org/2000/svg" width="480" height="240" viewBox="0 0 480 240">+ <rect width="480" height="240" rx="8" fill="#f1f5f9" />+ <text x="24" y="32" font-family="sans-serif" font-size="18" fill="#0f172a">Monthly revenue</text>+ <path d="M48 56V192H448" fill="none" stroke="#94a3b8" stroke-width="2" />+ <g fill="#6366f1">+ <rect x="72" y="144" width="48" height="48" rx="4" />+ <rect x="152" y="120" width="48" height="72" rx="4" />+ <rect x="232" y="96" width="48" height="96" rx="4" />+ <rect x="312" y="64" width="48" height="128" rx="4" />+ </g>+ <g font-family="sans-serif" font-size="12" fill="#475569" text-anchor="middle">+ <text x="96" y="216">Jan</text>+ <text x="176" y="216">Feb</text>+ <text x="256" y="216">Mar</text>+ <text x="336" y="216">Apr</text>+ </g>+</svg>diff --git a/web/.storybook/public/revenue-report.pdf b/web/.storybook/public/revenue-report.pdfnew file mode 100644index 0000000000..2c3179e9dc--- /dev/null+++ b/web/.storybook/public/revenue-report.pdf@@ -0,0 +1,32 @@+%PDF-1.4+1 0 obj+<< /Type /Catalog /Pages 2 0 R >>+endobj+2 0 obj+<< /Type /Pages /Kids [3 0 R] /Count 1 >>+endobj+3 0 obj+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 480 240] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>+endobj+4 0 obj+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>+endobj+5 0 obj+<< /Length 54 >>+stream+BT /F1 18 Tf 32 192 Td (Monthly revenue report) Tj ET+endstream+endobj+xref+0 6+0000000000 65535 f+0000000009 00000 n+0000000058 00000 n+0000000115 00000 n+0000000241 00000 n+0000000311 00000 n+trailer+<< /Size 6 /Root 1 0 R >>+startxref+414+%%EOFdiff --git a/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts b/web/src/__tests__/server/unit/loadTraceTranscript.servertest.tsindex fa9e417fa5..fa1cf35e79 100644--- a/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts+++ b/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts@@ -1,25 +1,12 @@import { beforeEach, expect, it, vi } from "vitest";-import type { Transcript } from "@langfuse/shared/src/server";import { assembleTranscript } from "@langfuse/shared/src/server/transcript/transcript";import { loadTraceTranscript } from "@/src/features/events/server/loadTraceTranscript";-import { normalizeSpanIO } from "@langfuse/shared/src/utils/normalized-io";-import type * as TranscriptThreads from "@langfuse/shared/src/server/transcript/threads";const mocks = vi.hoisted(() => ({assemble: vi.fn(),- splitTurn: vi.fn(),- splitTurnActual: vi.fn(),observations: vi.fn(),root: vi.fn(),}));-vi.mock(- "@langfuse/shared/src/server/transcript/threads",- async (importOriginal) => {- const actual = await importOriginal<typeof TranscriptThreads>();- mocks.splitTurnActual.mockImplementation(actual.splitTurn);- return { ...actual, splitTurn: mocks.splitTurn };- },-);vi.mock("@langfuse/shared/src/server", async () => {const { normalizeSpanIO } =await import("@langfuse/shared/src/utils/normalized-io");@@ -38,167 +25,40 @@ const trace = {traceId: "trace",timestamp: new Date(0),};-function fixture(- names = ["search", "weather"],- responses = ["weather", "search"],-) {- const provenance = {- traceId: "trace",- startTime: new Date(0),- endTime: null,- };- const transcript: Transcript = {- threads: [- {- conversationHistory: [],- currentTurn: {- nestingLevel: 0,- observations: [],- messages: [- {- ...provenance,- observationId: "generation",- role: "assistant",- source: "output",- parts: names.map((toolName, index) => ({- type: "tool-call",- toolName,- toolCallId: `call-${index}`,- input: {},- })),- },- ...responses.map((name, index) => ({- ...provenance,- observationId: `tool-${index}`,- role: "tool" as const,- source: "output" as const,- parts: [{ type: "text" as const, text: name }],- })),- ],- },- },- ],- };- mocks.assemble.mockReturnValue(transcript);- mocks.splitTurn.mockReturnValue(transcript.threads[0]);- mocks.observations.mockResolvedValue({- observations: [- {- ...provenance,- id: "generation",- type: "GENERATION",- input: [{ role: "user", content: "Run a tool" }],- output: [- {- role: "assistant",- parts: transcript.threads[0]!.currentTurn.messages[0]!.parts,- },- ],- metadata: {},- },- ...responses.map((name, index) => ({- ...provenance,- id: `tool-${index}`,- type: "TOOL",- name,- parentObservationId: null,- input: null,- output: name,- metadata: {},- })),- ],- totalCount: responses.length + 1,- });- return transcript;-}beforeEach(() => {vi.clearAllMocks();mocks.assemble.mockReset();mocks.observations.mockReset();mocks.root.mockReset();- mocks.splitTurn.mockReset().mockImplementation(mocks.splitTurnActual);-});--it("retains the source tool observation status on transcript messages", async () => {- const transcript = fixture(["search"], ["search"]);- mocks.observations.mockResolvedValue({- observations: [- {- id: "generation",- type: "GENERATION",- level: "DEFAULT",- statusMessage: null,- traceId: "trace",- },- {- id: "tool-0",- type: "TOOL",- level: "ERROR",- statusMessage: "Search timed out",- traceId: "trace",- },- ],- totalCount: 2,- });-- const result = await loadTraceTranscript(trace);-- expect(result.transcript?.threads[0]?.currentTurn.messages[1]).toMatchObject({- observationId: "tool-0",- level: "ERROR",- statusMessage: "Search timed out",- });- expect(transcript.threads[0]?.currentTurn.messages[1]).not.toHaveProperty(- "level",- );-});--it("does not copy generation errors onto unmatched tool calls", async () => {- const transcript = fixture(["search"], []);- mocks.observations.mockResolvedValue({- observations: [- {- id: "generation",- traceId: "trace",- type: "GENERATION",- level: "ERROR",- statusMessage: "Generation failed",- },- ],- totalCount: 1,- });- const result = await loadTraceTranscript(trace);- expect(result.transcript).toBe(transcript);- expect(- result.transcript?.threads[0]?.currentTurn.messages[0],- ).not.toHaveProperty("level");});-it.each(["ERROR", "WARNING"] as const)(- "recovers %s tool status with null output",- async (level) => {+it.each(+ ["ERROR", "WARNING"].flatMap((level) =>+ [null, "", []].map((output) => ({ level, output })),+ ),+)(+ "retains tool status with empty output without metadata recovery: %j",+ async ({ level, output }) => {const generation = {id: "generation",traceId: "trace",type: "GENERATION" as const,name: "generation",parentObservationId: null,+ nestingLevel: 0,startTime: new Date(0),endTime: new Date(1),input: [{ role: "user", content: "Run a tool" }],- output: [- {- role: "assistant",- parts: [- {- type: "tool-call",- toolName: "search",- toolCallId: "call-1",- input: {},- },- ],- },- ],+ output: {+ role: "assistant",+ tool_calls: [+ {+ id: "call-1",+ type: "function",+ function: { name: "search", arguments: "{}" },+ },+ ],+ },metadata: {},};const tool = {@@ -209,29 +69,24 @@ it.each(["ERROR", "WARNING"] as const)(parentObservationId: generation.id,startTime: new Date(2),input: {},- output: null,+ output,level,statusMessage: "Search unavailable",- metadata: { callID: "call-1" },+ metadata: { callID: "conflicting" },};const observations = [generation, tool];const snapshot = structuredClone(observations);- mocks.observations.mockResolvedValue({- observations,- totalCount: observations.length,- });- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });+ mocks.observations.mockResolvedValue({ observations, totalCount: 2 });+ mocks.assemble.mockImplementation(assembleTranscript);+ const result = await loadTraceTranscript(trace);expect(- result.transcript?.threads- .flatMap((thread) => thread.currentTurn.messages)- .find((message) => message.observationId === "tool"),+ result.transcript?.threads[0]?.currentTurn.messages.find(+ (message) => message.observationId === "tool",+ ),).toMatchObject({+ role: "tool",level,statusMessage: tool.statusMessage,- role: "tool",parts: [{type: "tool-result",@@ -245,260 +100,134 @@ it.each(["ERROR", "WARNING"] as const)(},);-it.each(["image", "media-token"])(- "attaches reversed nested %s results by recovered call ID",- async (scenario) => {- const outer = {- id: "outer",- traceId: "trace",- type: "GENERATION" as const,- name: "generation",- parentObservationId: null,- startTime: new Date(0),- endTime: new Date(1),- input: [{ role: "user", content: "Run tools" }],- output: null,- metadata: {},- };- const inner = {- ...outer,- id: "inner",- parentObservationId: outer.id,- startTime: new Date(1),- input: [- ...outer.input,+it("matches parallel same-name TOOL results by payload ID, not metadata", async () => {+ const generation = {+ id: "generation",+ traceId: "trace",+ type: "GENERATION" as const,+ name: "generation",+ parentObservationId: null,+ nestingLevel: 0,+ startTime: new Date(0),+ endTime: new Date(1),+ input: [{ role: "user", content: "Run tools" }],+ output: {+ role: "assistant",+ tool_calls: ["call-a", "call-b"].map((id) => ({+ id,+ type: "function",+ function: { name: "search", arguments: "{}" },+ })),+ },+ metadata: {},+ };+ const tools = ["call-b", "call-a"].map((id, index) => ({+ ...generation,+ id: `tool-${index}`,+ type: "TOOL" as const,+ name: "search",+ parentObservationId: generation.id,+ startTime: new Date(index + 2),+ input: {},+ output: { role: "tool", tool_call_id: id, content: `result-${id}` },+ metadata: { callID: "conflicting", toolCallId: "conflicting" },+ }));+ const observations = [generation, ...tools];+ const snapshot = structuredClone(observations);+ mocks.observations.mockResolvedValue({ observations, totalCount: 3 });+ mocks.assemble.mockImplementation(assembleTranscript);++ const result = await loadTraceTranscript(trace);+ const messages = result.transcript!.threads[0]!.currentTurn.messages;+ expect(messages.filter((message) => message.role === "tool")).toHaveLength(2);+ for (const tool of tools) {+ expect(+ messages.find((message) => message.observationId === tool.id),+ ).toMatchObject({+ parts: [{- role: "assistant",- content: [- {- type: "tool-call",- toolCallId: "call-a",- toolName: "search",- input: { query: "outer" },- },- ],+ type: "tool-result",+ toolCallId: tool.output.tool_call_id,+ output: { type: "text", value: tool.output.content },},],- };- const tools = [inner, outer].map((parent, index) => ({- ...outer,- id: `tool-${parent.id}`,- type: "TOOL" as const,- name: "search",- parentObservationId: parent.id,- startTime: new Date(index + 2),- input: { query: parent.id },- output:- scenario === "image"- ? [- {- type: "image_url",- image_url: { url: `https://example.com/${parent.id}.png` },- },- ]- : `@@@langfuseMedia:type=image/jpeg|id=media-${parent.id}|source=base64@@@`,- metadata: { callID: parent.id === "outer" ? "call-a" : "call-b" },- }));- const observations = [outer, inner, ...tools];- const snapshot = structuredClone(observations);- mocks.observations.mockResolvedValue({- observations,- totalCount: observations.length,- });- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,});- expect(result.transcript!.threads).toHaveLength(1);- const messages = result.transcript!.threads[0]!.currentTurn.messages;- for (const tool of tools) {- const callIndex = messages.findIndex(- (message) =>- message.source === "output" &&- message.parts.some(- (part) =>- part.type === "tool-call" &&- part.toolCallId === tool.metadata.callID,- ),- );- expect(callIndex).toBeGreaterThanOrEqual(0);- const response = messages[callIndex + 1];- expect(response).toMatchObject({ observationId: tool.id, role: "tool" });- expect(response!.parts).toEqual(- normalizeSpanIO({- input: undefined,- output: tool.output,- metadata: {},- }).messages.flatMap((message) => message.parts),- );- expect(response!.parts.some((part) => part.type === "file")).toBe(true);- expect(response!.parts.some((part) => part.type === "tool-result")).toBe(- false,- );- }- expect(observations).toEqual(snapshot);- },-);+ }+ expect(observations).toEqual(snapshot);+});-it.each(["missing", "idless"])(- "recovers a %s call despite duplicate observation rows",- async (scenario) => {- const input = { query: "search" };+it.each(["callID", "toolCallId"])(+ "does not synthesize generation calls from metadata.%s",+ async (metadataKey) => {const generation = {id: "generation",traceId: "trace",type: "GENERATION" as const,name: "generation",parentObservationId: null,+ nestingLevel: 0,startTime: new Date(0),endTime: new Date(1),- input: [{ role: "user", content: "Run a tool" }],- output:- scenario === "idless"- ? [- {- role: "assistant",- parts: [{ type: "tool-call", toolName: "search", input }],- },- ]- : null,- metadata: {},- };- const tool = {- ...generation,- id: "tool",- type: "TOOL" as const,- name: "search",- parentObservationId: generation.id,- startTime: new Date(2),- input,+ input: "Question",output: { answer: 42 },- metadata: { callID: "call-1" },+ metadata: {},};- const observations = [generation, { ...generation }, tool, { ...tool }];- const snapshot = structuredClone(observations);- mocks.observations.mockResolvedValue({- observations,- totalCount: observations.length,- });- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- const parts = result- .transcript!.threads.flatMap((thread) => thread.currentTurn.messages)- .flatMap((message) => message.parts);- expect(parts.filter((part) => part.type === "tool-call")).toMatchObject([- { toolCallId: "call-1" },- ]);- expect(parts.filter((part) => part.type === "tool-result")).toMatchObject([- { toolCallId: "call-1", output: { type: "json", value: { answer: 42 } } },- ]);- expect(observations).toEqual(snapshot);+ const observations = [+ generation,+ {+ ...generation,+ id: "tool",+ type: "TOOL" as const,+ name: "search",+ parentObservationId: generation.id,+ output: { result: 42 },+ metadata: { [metadataKey]: "call-1" },+ },+ ];+ mocks.observations.mockResolvedValue({ observations, totalCount: 2 });+ mocks.assemble.mockImplementation(assembleTranscript);++ const result = await loadTraceTranscript(trace);+ expect(result.transcript).toEqual(assembleTranscript([generation]));},);-it.each(- [- {- name: "image block",- output: [- {- type: "image_url",- image_url: { url: "https://example.com/image.png" },- },- ],- },- {- name: "file block",- output: [- {- type: "file",- data: "aGVsbG8=",- mediaType: "application/pdf",- filename: "result.pdf",- },- ],- },- {- name: "media token",- output:- "@@@langfuseMedia:type=image/jpeg|id=media-ref-image-1|source=base64@@@",- },- {- name: "text with media token",- output:- "Screenshot: @@@langfuseMedia:type=image/jpeg|id=media-ref-image-1|source=base64@@@",- },- ].flatMap((scenario) =>- [false, true].map((nativeCall) => ({ ...scenario, nativeCall })),- ),-)(- "preserves TOOL $name with native call=$nativeCall",- async ({ output, nativeCall }) => {- const input = { query: "screenshot" };- const generation = {- id: "generation",- traceId: "trace",- type: "GENERATION" as const,- name: "generation",- parentObservationId: null,- startTime: new Date(0),- endTime: new Date(1),- input: [{ role: "user", content: "Run a tool" }],- output: nativeCall- ? [- {- role: "assistant",- parts: [- {- type: "tool-call",- toolCallId: "call-1",- toolName: "search",- input,- },- ],- },- ]- : null,- metadata: {},- };- const tool = {+it.each([+ { structureCount: 10000, contentCount: 1000, cutoff: false },+ { structureCount: 10001, contentCount: 1000, cutoff: true },+ { structureCount: 10000, contentCount: 1001, cutoff: true },+])(+ "merges capped I/O with structure and reports cutoff: %j",+ async ({ structureCount, contentCount, cutoff }) => {+ const span = { id: "span", type: "SPAN" };+ const generation = { id: "generation", type: "GENERATION" };+ const tool = { id: "tool", type: "TOOL" };+ const generationWithContent = {...generation,- id: "tool",- type: "TOOL" as const,- name: "search",- parentObservationId: generation.id,- startTime: new Date(2),- input,- output,- metadata: { callID: "call-1" },+ input: "question",+ output: "answer",};- const observations = [generation, tool];- const snapshot = structuredClone(observations);- const expectedParts = normalizeSpanIO({- input: undefined,- output,- metadata: {},- }).messages.flatMap((message) => message.parts);- expect(expectedParts.some((part) => part.type === "file")).toBe(true);- mocks.observations.mockResolvedValue({ observations, totalCount: 2 });- const result = await loadTraceTranscript({+ mocks.observations+ .mockResolvedValueOnce({+ observations: [span, generation, tool],+ totalCount: structureCount,+ })+ .mockResolvedValueOnce({+ observations: [generationWithContent],+ totalCount: contentCount,+ });++ const result = await loadTraceTranscript(trace);++ expect(mocks.assemble).toHaveBeenCalledWith([span, generationWithContent]);+ expect(result.cutoff).toBe(cutoff);+ expect(mocks.observations).toHaveBeenNthCalledWith(1, trace);+ expect(mocks.observations).toHaveBeenNthCalledWith(2, {...trace,- recoverToolResponses: true,+ selectIOAndMetadata: true,+ types: ["GENERATION", "TOOL"],+ limit: 1000,});- const messages = result.transcript!.threads[0]!.currentTurn.messages;- expect(- messages- .filter((message) => message.observationId === tool.id)- .flatMap((message) => message.parts),- ).toEqual(expectedParts);- expect(- messages- .flatMap((message) => message.parts)- .filter((part) => part.type === "tool-call"),- ).toMatchObject([{ toolCallId: "call-1" }]);- expect(observations).toEqual(snapshot);},);@@ -570,181 +299,6 @@ it.each([},);-it.each([1, 2])(- "does not complete ambiguous ID-less calls with %i TOOL executions",- async (toolCount) => {- const input = { code: "return 42" };- const call = { type: "tool-call", toolName: "execute", input };- const generation = {- id: "generation",- traceId: "trace",- type: "GENERATION" as const,- name: "generation",- parentObservationId: null,- startTime: new Date(0),- endTime: new Date(1),- input: [{ role: "user", content: "Run a tool" }],- output: [- {- role: "assistant",- // Different raw encodings survive parser deduplication but normalize- // to identical arguments, so neither call uniquely identifies the TOOL.- parts:- toolCount === 1- ? [call, { ...call, input: JSON.stringify(input, null, 2) }]- : [call],- },- ],- metadata: {},- };- const observations = [- generation,- ...Array.from({ length: toolCount }, (_, index) => ({- ...generation,- id: `tool-${index}`,- type: "TOOL" as const,- name: "execute",- parentObservationId: generation.id,- startTime: new Date(index + 2),- input,- output: "42",- metadata: { callID: `call-${index}` },- })),- ];- mocks.observations.mockResolvedValue({- observations,- totalCount: observations.length,- });- mocks.assemble.mockImplementation(assembleTranscript);- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- const calls = result- .transcript!.threads[0]!.currentTurn.messages.flatMap(- (message) => message.parts,- )- .filter((part) => part.type === "tool-call");- const baseline = assembleTranscript([{ ...generation, nestingLevel: 0 }]);- const originalCalls = baseline!.threads[0]!.currentTurn.messages.flatMap(- (message) => message.parts,- ).filter((part) => part.type === "tool-call");- expect(calls).toEqual(originalCalls);- expect(calls.every((part) => part.toolCallId === null)).toBe(true);- },-);--it.each([- { name: "JSON", output: { answer: 42 } },- {- name: "reasoning",- output: [- {- role: "assistant",- content: [- { type: "reasoning", text: "Thinking", signature: "signature" },- ],- },- ],- },-])(- "preserves generation $name while recovering calls through real assembly",- async ({ output }) => {- const generation = {- id: "generation",- traceId: "trace",- type: "GENERATION" as const,- name: "generation",- parentObservationId: null,- startTime: new Date(0),- endTime: new Date(1),- input: [{ role: "user", content: "Run a tool" }],- output,- metadata: {},- };- const observations = [- generation,- {- ...generation,- id: "tool",- type: "TOOL" as const,- name: "execute",- parentObservationId: generation.id,- startTime: new Date(2),- input: { code: "return 42" },- output: "42",- metadata: { callID: "call-1" },- },- ];- const snapshot = structuredClone(observations);- const baseline = assembleTranscript([{ ...generation, nestingLevel: 0 }]);- mocks.observations.mockResolvedValue({ observations, totalCount: 2 });- mocks.assemble.mockImplementation(assembleTranscript);-- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- const originalParts = baseline!.threads[0]!.currentTurn.messages.filter(- (message) => message.source === "output",- ).flatMap((message) => message.parts);- const recoveredParts = result- .transcript!.threads[0]!.currentTurn.messages.filter(- (message) =>- message.observationId === generation.id &&- message.source === "output",- )- .flatMap((message) => message.parts);- expect(recoveredParts.filter((part) => part.type !== "tool-call")).toEqual(- originalParts,- );- expect(- recoveredParts.filter((part) => part.type === "tool-call"),- ).toMatchObject([{ toolCallId: "call-1" }]);- expect(observations).toEqual(snapshot);- },-);--it.each([- { structureCount: 10000, contentCount: 1000, cutoff: false },- { structureCount: 10001, contentCount: 1000, cutoff: true },- { structureCount: 10000, contentCount: 1001, cutoff: true },-])(- "merges capped I/O with structure and reports cutoff: %j",- async ({ structureCount, contentCount, cutoff }) => {- fixture();- const span = { id: "span", type: "SPAN" };- const generation = { id: "generation", type: "GENERATION" };- const tool = { id: "tool", type: "TOOL" };- const generationWithContent = {- ...generation,- input: "question",- output: "answer",- };- mocks.observations- .mockResolvedValueOnce({- observations: [span, generation, tool],- totalCount: structureCount,- })- .mockResolvedValueOnce({- observations: [generationWithContent],- totalCount: contentCount,- });-- const result = await loadTraceTranscript(trace);-- expect(mocks.assemble).toHaveBeenCalledWith([span, generationWithContent]);- expect(result.cutoff).toBe(cutoff);- expect(mocks.observations).toHaveBeenNthCalledWith(1, trace);- expect(mocks.observations).toHaveBeenNthCalledWith(2, {- ...trace,- selectIOAndMetadata: true,- types: ["GENERATION", "TOOL"],- limit: 1000,- });- },-);-it("falls back to seeded JSON chat messages with an empty-string root parent", async () => {const root = {id: "t-seed-media-image-only",@@ -794,8 +348,10 @@ it("falls back to seeded JSON chat messages with an empty-string root parent", a]);});-it("does not fetch root I/O when an existing transcript is available", async () => {- const original = fixture();+it("returns the shared transcript unchanged without fetching root I/O", async () => {+ const original = { threads: [] };+ mocks.assemble.mockReturnValue(original);+ mocks.observations.mockResolvedValue({ observations: [], totalCount: 0 });const result = await loadTraceTranscript({...trace,fallbackToRootIO: true,@@ -838,456 +394,3 @@ it("leaves the transcript empty when the root has no input/output", async () =>.transcript,).toBeNull();});-it("leaves mixed text/data responses unchanged", async () => {- const original = fixture(["search"], ["search"]);- original.threads[0]!.currentTurn.messages[1]!.parts = [- { type: "text", text: "result" },- { type: "data", value: { answer: 42 } },- ];- const snapshot = structuredClone(original);-- expect(- (await loadTraceTranscript({ ...trace, recoverToolResponses: true }))- .transcript,- ).toEqual(snapshot);- expect(original).toEqual(snapshot);-});--it("preserves explicit results that conflict with TOOL metadata", async () => {- const observations = [- {- id: "generation",- traceId: "trace",- type: "GENERATION",- name: "generation",- parentObservationId: null,- startTime: new Date(0),- endTime: new Date(1),- input: null,- output: [- {- role: "assistant",- parts: [- {- type: "tool-call",- toolCallId: "explicit",- toolName: "search",- input: {},- },- ],- },- ],- metadata: {},- },- {- id: "tool",- traceId: "trace",- type: "TOOL",- name: "search",- parentObservationId: "generation",- startTime: new Date(2),- endTime: new Date(3),- input: {},- output: { role: "tool", tool_call_id: "explicit", content: "result" },- metadata: { callID: "conflicting" },- },- ];- const snapshot = structuredClone(observations);- mocks.observations.mockResolvedValue({ observations, totalCount: 2 });- mocks.assemble.mockImplementation(assembleTranscript);-- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });-- const parts = result.transcript!.threads[0]!.currentTurn.messages.flatMap(- (message) => message.parts,- );- expect(parts.filter((part) => part.type === "tool-result")).toMatchObject([- { toolCallId: "explicit", output: { type: "text", value: "result" } },- ]);- expect(- parts.some(- (part) => part.type === "tool-call" && part.toolCallId === "conflicting",- ),- ).toBe(false);- expect(observations).toEqual(snapshot);-});--it.each([- "missing-generation-call",- "existing-generation-call",- "idless-generation-call",- "existing-tool-result",-])(- "pairs TOOL metadata call IDs through real assembly: %s",- async (scenario) => {- const input = { code: "return 42" };- const output = { title: "Result", output: "42" };- const observations = [- {- id: "generation",- traceId: "trace",- type: "GENERATION" as const,- name: "generation",- parentObservationId: null,- startTime: new Date(0),- endTime: new Date(1),- input: [{ role: "user", content: "Run a tool" }],- output:- scenario === "existing-generation-call" ||- scenario === "idless-generation-call"- ? [- {- role: "assistant",- parts: [- {- type: "tool-call",- toolCallId:- scenario === "idless-generation-call" ? null : "call-1",- toolName: "execute",- input,- },- ],- },- ]- : [{ role: "assistant", content: "" }],- metadata: {},- },- {- id: "tool",- traceId: "trace",- type: "TOOL" as const,- name: "execute",- parentObservationId: "generation",- startTime: new Date(2),- endTime: new Date(3),- input,- output:- scenario === "existing-tool-result"- ? {- role: "tool",- name: "execute",- tool_call_id: "call-1",- content: output,- }- : output,- metadata: { callID: "call-1" },- },- ];- const snapshot = structuredClone(observations);- mocks.assemble.mockImplementation(assembleTranscript);- mocks.observations.mockResolvedValue({- observations,- totalCount: observations.length,- });- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- const messages = result.transcript!.threads[0]!.currentTurn.messages;- expect(- messages- .flatMap((message) => message.parts)- .filter((part) => part.type === "tool-call"),- ).toMatchObject([- { type: "tool-call", toolCallId: "call-1", toolName: "execute", input },- ]);- expect(- messages.find((message) => message.observationId === "tool"),- ).toMatchObject({- role: "tool",- source: "output",- observationId: "tool",- startTime: new Date(2),- endTime: new Date(3),- parts: [- {- type: "tool-result",- toolCallId: "call-1",- toolName: "execute",- output: { type: "json", value: output },- },- ],- });- expect(observations).toEqual(snapshot);- expect((await loadTraceTranscript(trace)).transcript).toEqual(- assembleTranscript(- observations.map((observation) => ({- ...observation,- nestingLevel: 0,- })),- ),- );- },-);--it.each(["duplicate-call-id", "different-trace", "non-generation-parent"])(- "does not reconstruct ambiguous TOOL metadata: %s",- async (scenario) => {- const original = fixture(["execute"], ["execute"]);- const generation = {- id: "generation",- traceId: "trace",- type: "GENERATION",- name: "generation",- parentObservationId: null,- startTime: new Date(0),- endTime: null,- input: null,- output: null,- metadata: {},- };- const tool = {- ...generation,- id: "tool",- type: "TOOL",- name: "execute",- parentObservationId: "generation",- metadata: { callID: "call-1" },- input: { code: "return 42" },- output: { answer: 42 },- };- if (scenario === "different-trace") generation.traceId = "other-trace";- if (scenario === "non-generation-parent") generation.type = "SPAN";- const observations =- scenario === "duplicate-call-id"- ? [generation, tool, { ...tool, id: "tool-duplicate" }]- : [generation, tool];- mocks.observations.mockResolvedValue({- observations,- totalCount: observations.length,- });- await loadTraceTranscript({ ...trace, recoverToolResponses: true });- expect(original.threads[0]!.currentTurn.messages[1]!.parts[0]?.type).toBe(- "text",- );- },-);-it("opts into provenance-based reverse multi-call responses without mutating assembly", async () => {- const original = fixture();- const snapshot = structuredClone(original);- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- expect(- result.transcript?.threads[0]?.currentTurn.messages- .slice(1)- .map((m) => m.parts),- ).toEqual([- [- {- type: "tool-result",- toolCallId: "call-1",- toolName: "weather",- output: { type: "content", value: [{ type: "text", text: "weather" }] },- },- ],- [- {- type: "tool-result",- toolCallId: "call-0",- toolName: "search",- output: { type: "content", value: [{ type: "text", text: "search" }] },- },- ],- ]);- expect(original).toEqual(snapshot);- expect(result.transcript?.threads[0]?.currentTurn.messages[1]).toMatchObject({- observationId: "tool-0",- traceId: "trace",- source: "output",- startTime: new Date(0),- endTime: null,- });- expect(mocks.observations).toHaveBeenCalledTimes(2);-});-it("leaves the default transcript unchanged", async () => {- const original = fixture();- expect((await loadTraceTranscript(trace)).transcript).toBe(original);-});-it.each([- { output: { answer: 42 } },- { output: [{ answer: 42 }] },- { output: null },-])(- "pairs JSON-only TOOL output using observation provenance: %j",- async ({ output }) => {- const original = fixture(["search"], ["search"]);- original.threads[0]!.currentTurn.messages[1]!.parts = [- { type: "data", value: output },- ];- const snapshot = structuredClone(original);- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- expect(- result.transcript?.threads[0]?.currentTurn.messages[1]?.parts,- ).toEqual([- {- type: "tool-result",- toolCallId: "call-0",- toolName: "search",- output: { type: "content", value: [{ type: "data", value: output }] },- },- ]);- expect(original).toEqual(snapshot);- expect((await loadTraceTranscript(trace)).transcript).toBe(original);- },-);--it("preserves multiple JSON parts as typed content", async () => {- const original = fixture(["search"], ["search"]);- original.threads[0]!.currentTurn.messages[1]!.parts = [- { type: "data", value: { answer: 42 } },- { type: "data", value: { confidence: 0.9 } },- ];- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- expect(result.transcript?.threads[0]?.currentTurn.messages[1]?.parts).toEqual(- [- {- type: "tool-result",- toolCallId: "call-0",- toolName: "search",- output: {- type: "content",- value: [- { type: "data", value: { answer: 42 } },- { type: "data", value: { confidence: 0.9 } },- ],- },- },- ],- );-});--it.each([- "input",- "wrong-observation-type",- "ambiguous-call",- "explicit-result",-])("does not infer JSON tool results for %s", async (scenario) => {- const original = fixture(["search"], ["search"]);- const messages = original.threads[0]!.currentTurn.messages;- messages[1]!.parts = [{ type: "data", value: { answer: 42 } }];- if (scenario === "input") messages[1]!.source = "input";- if (scenario === "wrong-observation-type") {- mocks.observations.mockResolvedValue({- observations: [- {- id: "tool-0",- traceId: "trace",- type: "GENERATION",- name: "search",- input: [{ role: "user", content: "Run a tool" }],- },- ],- totalCount: 1,- });- }- if (scenario === "ambiguous-call") {- messages[0]!.parts.push({- type: "tool-call",- toolCallId: "other",- toolName: "search",- input: {},- });- }- if (scenario === "explicit-result") {- messages.push({- ...messages[1]!,- parts: [- {- type: "tool-result",- toolCallId: "call-0",- output: { type: "text", value: "explicit" },- },- ],- });- }- const snapshot = structuredClone(original);- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- expect(result.transcript).toEqual(snapshot);- expect(original).toEqual(snapshot);-});-it.each([- { names: ["search", "search"], responses: ["search"] },- { names: ["search"], responses: ["search", "search"] },-])(- "leaves ambiguous names and duplicate responses separate: %j",- async ({ names, responses }) => {- const original = fixture(names, responses);- expect(- (await loadTraceTranscript({ ...trace, recoverToolResponses: true }))- .transcript,- ).toEqual(original);- },-);-it("preserves explicit result precedence", async () => {- const original = fixture(["search"], ["search"]);- original.threads[0]!.currentTurn.messages.push({- ...original.threads[0]!.currentTurn.messages[1]!,- parts: [- {- type: "tool-result",- toolCallId: "call-0",- output: { type: "text", value: "explicit" },- },- ],- });- expect(- (await loadTraceTranscript({ ...trace, recoverToolResponses: true }))- .transcript,- ).toEqual(original);-});-it("requires exact response trace provenance even for adjacent single calls", async () => {- const original = fixture(["search"], ["search"]);- original.threads[0]!.currentTurn.messages[1]!.traceId = "other-trace";- expect(- (await loadTraceTranscript({ ...trace, recoverToolResponses: true }))- .transcript,- ).toEqual(original);-});-it.each(["missing-observation", "conflicting-name", "reused-id"])(- "does not infer %s",- async (scenario) => {- const original = fixture(["search"], ["search"]);- const messages = original.threads[0]!.currentTurn.messages;- if (scenario === "missing-observation")- messages[1]!.observationId = "missing";- if (scenario === "conflicting-name") {- const part = messages[0]!.parts[0]!;- if (part.type === "tool-call") part.toolName = "weather";- }- if (scenario === "reused-id")- messages[0]!.parts.push({ ...messages[0]!.parts[0]! });- expect(- (await loadTraceTranscript({ ...trace, recoverToolResponses: true }))- .transcript,- ).toEqual(original);- },-);-it("leaves history untouched when it shares current-turn message references", async () => {- const original = fixture(["search"], ["search"]);- original.threads[0]!.conversationHistory =- original.threads[0]!.currentTurn.messages;- const result = await loadTraceTranscript({- ...trace,- recoverToolResponses: true,- });- expect(- result.transcript?.threads[0]?.conversationHistory[1]?.parts[0]?.type,- ).toBe("text");- expect(- result.transcript?.threads[0]?.currentTurn.messages[1]?.parts[0]?.type,- ).toBe("tool-result");-});diff --git a/web/src/features/events/server/eventsRouter.ts b/web/src/features/events/server/eventsRouter.tsindex ea452a74df..1892d7a025 100644--- a/web/src/features/events/server/eventsRouter.ts+++ b/web/src/features/events/server/eventsRouter.ts@@ -445,7 +445,6 @@ export const eventsRouter = createTRPCRouter({projectId: zodSchema.string(),traceId: zodSchema.string(),timestamp: zodSchema.date().optional(),- recoverToolResponses: zodSchema.boolean().default(false),fallbackToRootIO: zodSchema.boolean().default(false),}),)@@ -494,7 +493,6 @@ export const eventsRouter = createTRPCRouter({projectId: input.projectId,traceId: input.traceId,timestamp,- recoverToolResponses: input.recoverToolResponses,fallbackToRootIO: input.fallbackToRootIO,});},diff --git a/web/src/features/events/server/loadTraceTranscript.ts b/web/src/features/events/server/loadTraceTranscript.tsindex f15280c928..07646e677d 100644--- a/web/src/features/events/server/loadTraceTranscript.ts+++ b/web/src/features/events/server/loadTraceTranscript.ts@@ -1,4 +1,3 @@-import { isEqual, partition } from "lodash";import {assembleTranscript,getObservationByIdFromEventsTable,@@ -8,30 +7,14 @@ import {normalizeIO,type Transcript,} from "@langfuse/shared/src/server";-import {- normalizeSpanIO,- type NormalizedMessage,- type ToolCallPart,-} from "@langfuse/shared/src/utils/normalized-io";-import {- append,- createThread,- findThread,- messageKey,- splitTurn,- type ThreadState,-} from "@langfuse/shared/src/server/transcript/threads";-import { createToolCallRegistry } from "@langfuse/shared/src/server/transcript/tool-calls";/** Generations and tools whose I/O one transcript reads at most. */const MAX_TRANSCRIPT_OBSERVATIONS = 1_000;-const TRANSCRIPT_OBSERVATION_TYPES = new Set(["GENERATION", "TOOL"]);type Observation = Awaited<ReturnType<typeof getObservationsForTraceFromEventsTable>>["observations"][number];-type SessionTranscript = Omit<Transcript, "threads"> & {threads: Array<Omit<Transcript["threads"][number], "currentTurn"> & {@@ -50,30 +33,15 @@ type SessionTranscript = Omit<Transcript, "threads"> & {>;};-/**- * Load a trace's observations from the events table, walk them in trace tree- * order, and assemble the transcript of its generations and tools.- */+/** Load observations in trace tree order and assemble their transcript. */export async function loadTraceTranscript(trace: {projectId: string;traceId: string;/** Trace timestamp; observations from one hour before it onwards are read. */timestamp: Date;- /**- * For session rendering, recover calls from TOOL metadata and pair text-only- * or JSON-only responses when observation names uniquely identify their calls.- * Ambiguous responses remain unchanged. Disabled by default.- */- recoverToolResponses?: boolean;/** Use root observation I/O when no generation/tool transcript exists. */fallbackToRootIO?: boolean;-}): Promise<{- transcript: SessionTranscript | null;- /** The trace has more observations than the transcript could read. */- cutoff: boolean;-}> {- // The structure of every observation orders the walk; only generations and- // tools carry I/O, which is what the transcript reads.+}): Promise<{ transcript: SessionTranscript | null; cutoff: boolean }> {const [structure, content] = await Promise.all([getObservationsForTraceFromEventsTable(trace),getObservationsForTraceFromEventsTable({@@ -88,23 +56,41 @@ export async function loadTraceTranscript(trace: {// them out beats rendering them as empty turns. `cutoff` tells the caller.const observations = structure.observations.flatMap((o) => {const withContent = contentById.get(o.id);- if (withContent) return [withContent];+ if (withContent) {+ // Status-only executions still need a response row. Use normal assembly+ // matching without recovering IDs from metadata or inventing calls.+ if (+ withContent.type === "TOOL" &&+ (withContent.level === "ERROR" ||+ withContent.level === "WARNING" ||+ withContent.statusMessage) &&+ normalizeIO({+ kind: "io",+ io: {+ input: undefined,+ output: withContent.output,+ metadata: withContent.metadata,+ },+ }).messages.every((message) => message.parts.length === 0)+ ) {+ return [+ {+ ...withContent,+ output: {+ type: "tool-result",+ toolCallId: null,+ toolName: withContent.name,+ output: null,+ isError: withContent.level === "ERROR",+ },+ },+ ];+ }+ return [withContent];+ }return TRANSCRIPT_OBSERVATION_TYPES.has(o.type) ? [] : [o];});- let transcript = (() => {- if (!trace.recoverToolResponses) {- return assembleTranscript(orderObservations(observations));- }- const recovered = recoverToolCalls(observations);- return pairToolResponses(- assembleSessionTraceTranscript(- orderObservations(recovered.observations),- recovered.transformOutput,- recovered.mediaCallIds,- ),- recovered.observationsByTraceAndId,- );- })();+ let transcript = assembleTranscript(orderObservations(observations));if (!transcript && trace.fallbackToRootIO) {const root = structure.observations.find((observation) =>@@ -151,12 +137,11 @@ export async function loadTraceTranscript(trace: {}}}- const cutoff =- structure.totalCount > MAX_OBSERVATIONS_PER_TRACE ||- content.totalCount > MAX_TRANSCRIPT_OBSERVATIONS;return {transcript: withToolObservationStatus(transcript, contentById),- cutoff,+ cutoff:+ structure.totalCount > MAX_OBSERVATIONS_PER_TRACE ||+ content.totalCount > MAX_TRANSCRIPT_OBSERVATIONS,};}@@ -195,435 +180,3 @@ function withToolObservationStatus(})),};}--// TODO: Look into moving session tool recovery into shared transcript assembly-// so normalized-output recovery does not need separate assembly orchestration.-function assembleSessionTraceTranscript(- observations: Array<Observation & { nestingLevel: number }>,- transformOutput: (- observation: Observation,- messages: NormalizedMessage[],- ) => NormalizedMessage[],- mediaCallIds: Map<string, string>,-): Transcript | null {- const threads: ThreadState[] = [];- const toolCalls = createToolCallRegistry();- for (const observation of observations.filter(- (- observation,- ): observation is Observation & { nestingLevel: number; traceId: string } =>- (observation.type === "GENERATION" || observation.type === "TOOL") &&- observation.traceId !== null,- )) {- const { messages } = normalizeSpanIO({- input: observation.type === "TOOL" ? undefined : observation.input,- output: observation.output,- metadata: observation.metadata,- });- const [inputMessages, outputMessages] = partition(- messages,- (message) => message.source === "input",- );- if (- observation.type === "TOOL" &&- outputMessages.every((message) => message.parts.length === 0) &&- (observation.level === "ERROR" ||- observation.level === "WARNING" ||- observation.statusMessage)- ) {- outputMessages.push({- role: "tool",- source: "output",- parts: [{ type: "data", value: null }],- });- }- const [input, output] = partition(- [...inputMessages, ...transformOutput(observation, outputMessages)].map(- (message) => ({ message, key: messageKey(message) }),- ),- ({ message }) => message.source === "input",- );- if (observation.type === "TOOL") {- const callId = mediaCallIds.get(- JSON.stringify([observation.traceId, observation.id]),- );- if (callId) {- const marker: NormalizedMessage = {- role: "tool",- source: "output",- parts: [- {- type: "tool-result",- toolCallId: callId,- output: { type: "json", value: null },- },- ],- };- // Let the registry resolve identity and placement, then retain the- // original media parts instead of exposing a JSON-only result marker.- toolCalls.attachToolOutput(observation, [- { message: marker, key: messageKey(marker) },- ]);- for (const thread of threads) {- const response = thread.messages.find(- (message) =>- message.observationId === observation.id &&- message.traceId === observation.traceId &&- message.parts.some(- (part) =>- part.type === "tool-result" && part.toolCallId === callId,- ),- );- if (response) {- response.parts = output.flatMap(({ message }) => message.parts);- break;- }- }- continue;- }- toolCalls.attachToolOutput(observation, output);- continue;- }- if (input.length === 0 && output.length === 0) continue;- let thread = findThread(threads, input);- if (!thread) {- thread = createThread();- threads.push(thread);- }- append(thread, observation, input, output, toolCalls);- }- return threads.length ? { threads: threads.map(splitTurn) } : null;-}--function recoverToolCalls(observations: Observation[]) {- const observationsByTraceAndId = new Map<- Observation["traceId"],- Map<string, Observation>- >();- const callsByParent = new Map<string, ToolCallPart[]>();- const uniqueObservations: Observation[] = [];- const toolsByObservation = new Map<string, string>();- const callCountsByTrace = new Map<- Observation["traceId"],- Map<string, number>- >();- for (const observation of observations) {- let byId = observationsByTraceAndId.get(observation.traceId);- if (!byId) {- byId = new Map();- observationsByTraceAndId.set(observation.traceId, byId);- }- // Preserve the first match, as the previous array lookup did.- if (byId.has(observation.id)) continue;- byId.set(observation.id, observation);- uniqueObservations.push(observation);-- if (observation.type !== "TOOL") continue;- const callId = observation.metadata?.callID;- if (typeof callId !== "string" || !callId) continue;- let counts = callCountsByTrace.get(observation.traceId);- if (!counts) {- counts = new Map();- callCountsByTrace.set(observation.traceId, counts);- }- counts.set(callId, (counts.get(callId) ?? 0) + 1);- }-- // Some plugins omit calls from generation output. A TOOL's unique call ID- // and same-trace generation parent recover that anchor without name guessing.- for (const observation of uniqueObservations) {- if (- observation.type !== "TOOL" ||- !observation.name ||- !observation.parentObservationId- )- continue;- const callId = observation.metadata?.callID;- if (typeof callId !== "string" || !callId) continue;- if (callCountsByTrace.get(observation.traceId)?.get(callId) !== 1) continue;- const parent = observationsByTraceAndId- .get(observation.traceId)- ?.get(observation.parentObservationId);- if (parent?.type !== "GENERATION") continue;- const outputParts = normalizeSpanIO({- input: undefined,- output: observation.output,- metadata: undefined,- }).messages.flatMap((message) => message.parts);- const explicitResults = outputParts.flatMap((part) =>- part.type === "tool-result" && part.toolCallId ? [part] : [],- );- if (explicitResults.some((part) => part.toolCallId !== callId)) continue;- const normalized = normalizeSpanIO({- input: undefined,- metadata: undefined,- output: {- type: "tool-call",- toolCallId: callId,- toolName: observation.name,- input: observation.input,- },- });- const call = normalized.messages- .flatMap((message) => message.parts)- .find((part): part is ToolCallPart => part.type === "tool-call");- if (!call) continue;- const parentKey = JSON.stringify([parent.traceId, parent.id]);- const calls = callsByParent.get(parentKey);- if (calls) calls.push(call);- else callsByParent.set(parentKey, [call]);- if (!explicitResults.length) {- toolsByObservation.set(- JSON.stringify([observation.traceId, observation.id]),- callId,- );- }- }-- const resolvedCallIds = new Set<string>();- const mediaCallIds = new Map<string, string>();- const transformOutput = (- observation: Observation,- messages: NormalizedMessage[],- ): NormalizedMessage[] => {- if (observation.type === "TOOL") {- const callId = toolsByObservation.get(- JSON.stringify([observation.traceId, observation.id]),- );- if (- !callId ||- !resolvedCallIds.has(JSON.stringify([observation.traceId, callId]))- )- return messages;- if (- observation.output === null &&- (observation.level === "ERROR" ||- observation.level === "WARNING" ||- observation.statusMessage)- ) {- return [- {- role: "tool",- source: "output",- parts: [- {- type: "tool-result",- toolCallId: callId,- toolName: observation.name ?? "Tool",- output: { type: "json", value: null },- isError: observation.level === "ERROR",- },- ],- },- ];- }- const parts = messages.flatMap((message) => message.parts);- // Keep the session media rendering path when metadata recovers a call- // whose response contains media or other individually rendered parts.- if (!parts.length) return messages;- if (- !parts.every((part) => part.type === "text") &&- !parts.every((part) => part.type === "data")- ) {- mediaCallIds.set(- JSON.stringify([observation.traceId, observation.id]),- callId,- );- return messages;- }- return normalizeSpanIO({- input: undefined,- metadata: undefined,- output: {- role: "tool",- name: observation.name,- tool_call_id: callId,- content: observation.output,- },- }).messages;- }- const calls = callsByParent.get(- JSON.stringify([observation.traceId, observation.id]),- );- if (!calls || observation.type !== "GENERATION") return messages;- const existingIds = new Set(- messages.flatMap((message) =>- message.parts.flatMap((part) =>- part.type === "tool-call" ? [part.toolCallId] : [],- ),- ),- );- const idlessCalls = messages.flatMap((message) =>- message.parts.filter(- (part): part is ToolCallPart =>- part.type === "tool-call" && !part.toolCallId,- ),- );- const replacements = new Map<ToolCallPart, ToolCallPart>();- const missingCalls: ToolCallPart[] = [];- for (const call of calls) {- if (existingIds.has(call.toolCallId)) {- resolvedCallIds.add(- JSON.stringify([observation.traceId, call.toolCallId]),- );- continue;- }- const matches = idlessCalls.filter(- (existing) =>- existing.toolName === call.toolName &&- isEqual(existing.input, call.input),- );- // Ambiguous existing calls cannot safely be completed or duplicated.- if (matches.length > 0) {- const recoveredMatches = calls.filter(- (candidate) =>- candidate.toolName === call.toolName &&- isEqual(candidate.input, call.input),- );- if (matches.length === 1 && recoveredMatches.length === 1) {- replacements.set(matches[0]!, call);- resolvedCallIds.add(- JSON.stringify([observation.traceId, call.toolCallId]),- );- }- continue;- }- missingCalls.push(call);- resolvedCallIds.add(- JSON.stringify([observation.traceId, call.toolCallId]),- );- }- if (!replacements.size && !missingCalls.length) return messages;- const recoveredMessages = messages.map((message) => ({- ...message,- parts: message.parts.map((part) => {- if (part.type !== "tool-call") return part;- const replacement = replacements.get(part);- return replacement- ? { ...part, toolCallId: replacement.toolCallId }- : part;- }),- }));- return missingCalls.length- ? [- ...recoveredMessages,- { role: "assistant", source: "output", parts: missingCalls },- ]- : recoveredMessages;- };- return {- observations: uniqueObservations,- observationsByTraceAndId,- transformOutput,- mediaCallIds,- };-}--function pairToolResponses(- transcript: Transcript | null,- observationsByTraceAndId: Map<- Observation["traceId"],- Map<string, Observation>- >,-): Transcript | null {- // TODO: Evaluate moving general pairing into shared transcript assembly once- // it preserves resolved call identity for text-only and JSON-only tool responses.- if (!transcript) return null;- return {- ...transcript,- threads: transcript.threads.map((thread) => {- // Historical messages are already established context, not recovery targets.- const messages = thread.currentTurn.messages;- const callCounts = new Map<string, number>();- for (const message of messages) {- for (const part of message.parts) {- if (part.type !== "tool-call" || !part.toolCallId) continue;- callCounts.set(- part.toolCallId,- (callCounts.get(part.toolCallId) ?? 0) + 1,- );- }- }- const explicitIds = new Set(- messages.flatMap((message) =>- message.parts.flatMap((part) =>- part.type === "tool-result" && part.toolCallId- ? [part.toolCallId]- : [],- ),- ),- );- const candidates = new Map<number, ToolCallPart>();- const candidateCounts = new Map<string, number>();- const anchorCallsByName = new Map<string, ToolCallPart | null>();- for (const [index, message] of messages.entries()) {- if (message.role !== "tool") {- // Name matching is limited to the immediately preceding assistant- // output and its consecutive tool responses.- anchorCallsByName.clear();- if (message.role === "assistant" && message.source === "output") {- for (const part of message.parts) {- if (part.type !== "tool-call") continue;- // A repeated name cannot identify one call unambiguously.- anchorCallsByName.set(- part.toolName,- anchorCallsByName.has(part.toolName) ? null : part,- );- }- }- continue;- }- if (- message.source !== "output" ||- !message.parts.length ||- (!message.parts.every((part) => part.type === "text") &&- !message.parts.every((part) => part.type === "data"))- )- continue;- const observation = observationsByTraceAndId- .get(message.traceId)- ?.get(message.observationId);- // Provenance must identify a TOOL; positional guessing cannot override it.- if (observation?.type !== "TOOL" || !observation.name) continue;- const call = anchorCallsByName.get(observation.name);- if (!call) continue;- if (!call.toolCallId) continue;- if (explicitIds.has(call.toolCallId)) continue;- if (callCounts.get(call.toolCallId) !== 1) continue;- candidates.set(index, call);- candidateCounts.set(- call.toolCallId,- (candidateCounts.get(call.toolCallId) ?? 0) + 1,- );- }- return {- ...thread,- currentTurn: {- ...thread.currentTurn,- messages: messages.map((message, index) => {- const call = candidates.get(index);- if (!call?.toolCallId) return message;- if (candidateCounts.get(call.toolCallId) !== 1) return message;- const output = {- type: "content" as const,- value: message.parts.filter(- (part) => part.type === "text" || part.type === "data",- ),- };- return {- ...message,- parts: [- {- type: "tool-result" as const,- toolCallId: call.toolCallId,- toolName: call.toolName,- output,- },- ],- };- }),- },- };- }),- };-}diff --git a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsxindex c70f7eb9ee..bb23e8708c 100644--- a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx+++ b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx@@ -108,7 +108,7 @@ export function ConnectedModernSessionBodyTimeline({projectId,traceId: trace.id,timestamp: trace.timestamp,- recoverToolResponses: true,+ fallbackToRootIO: true,}) !== undefined,);const searchQuery = allTranscriptsLoadeddiff --git a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/SessionConversationalView.stories.tsx b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/SessionConversationalView.stories.tsxindex cf466e85fe..fbc521d2f9 100644--- a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/SessionConversationalView.stories.tsx+++ b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/SessionConversationalView.stories.tsx@@ -2041,12 +2041,141 @@ const manySimpleTurnsWorkflow: WorkflowFixture[] = Array.from(},);+const toolContentExamples = [+ {+ name: "describe_chart",+ content: [{ type: "text", text: "Monthly revenue increased." }],+ },+ {+ name: "get_chart_data",+ content: [{ points: 12, metric: "monthly-revenue" }],+ },+ {+ name: "download_chart",+ content: [+ {+ type: "image_url",+ image_url: { url: "/revenue-chart.svg" },+ },+ ],+ },+ {+ name: "inspect_chart",+ content: [+ { type: "vendor-chart", value: { chartId: "chart-annotations" } },+ ],+ },+ {+ name: "download_named_chart",+ content: [+ {+ type: "file",+ mediaType: "image/svg+xml",+ filename: "revenue.svg",+ data: { type: "url", url: "/revenue-chart.svg" },+ },+ ],+ },+ {+ name: "download_svg_chart",+ content: [+ {+ type: "file",+ mediaType: "image/svg+xml",+ data: { type: "url", url: "/revenue-chart.svg" },+ },+ ],+ },+ {+ name: "download_report",+ content: [+ {+ type: "file",+ mediaType: "application/pdf",+ data: { type: "url", url: "/revenue-report.pdf" },+ },+ ],+ },+ {+ name: "get_unknown_attachment",+ content: [{ type: "file", file: { file_id: "revenue-attachment" } }],+ },+ {+ name: "generate_chart",+ content: [+ { type: "text", text: "Monthly revenue chart generated." },+ { points: 24, metric: "mixed-revenue" },+ {+ type: "image_url",+ image_url: { url: "/revenue-chart.svg" },+ },+ { type: "vendor-chart", value: { chartId: "chart-revenue" } },+ ],+ },+];++const mixedToolContentWorkflow: WorkflowFixture[] = [+ {+ trace: {+ ...trace,+ id: "mixed-tool-content-story",+ name: "Generate chart with attachments",+ observationCount: toolContentExamples.length + 1,+ },+ turnNumber: 1,+ observations: [+ {+ id: "chart-generation",+ traceId: "mixed-tool-content-story",+ parentObservationId: null,+ type: "GENERATION",+ name: "Plan chart",+ startTime: trace.timestamp,+ endTime: new Date(trace.timestamp.getTime() + 1000),+ environment: "storybook",+ metadata: {},+ input: JSON.stringify([+ { role: "user", content: "Generate a chart of monthly revenue." },+ ]),+ output: JSON.stringify({+ role: "assistant",+ tool_calls: toolContentExamples.map(({ name }) => ({+ id: `${name}-call`,+ type: "function",+ function: {+ name,+ arguments: '{"metric":"revenue"}',+ },+ })),+ }),+ },+ ...toolContentExamples.map(({ name, content }, index) => ({+ id: `${name}-tool`,+ traceId: "mixed-tool-content-story",+ parentObservationId: "chart-generation",+ type: "TOOL" as const,+ name,+ startTime: new Date(trace.timestamp.getTime() + (index + 1) * 1000),+ endTime: new Date(trace.timestamp.getTime() + (index + 2) * 1000),+ environment: "storybook",+ metadata: {},+ input: JSON.stringify({ metric: "revenue" }),+ output: JSON.stringify({+ role: "tool",+ content,+ }),+ })),+ ],+ },+];+const workflowTranscripts = new Map([supportAgentWorkflow,codingAgentWorkflow,langfuseAssistantWorkflow,manySimpleTurnsWorkflow,+ mixedToolContentWorkflow,].map((workflow) => [workflow,workflow.map((item): WorkflowTrace => {@@ -2525,6 +2654,70 @@ const meta = preview.meta({});export default meta;+export const MixedToolContent = meta.story({+ name: "(Test) Mixed Tool Content",+ args: { workflowTraces: mixedToolContentWorkflow },+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement);+ const timeline = within(+ canvas.getByLabelText("Session conversation timeline"),+ );+ await userEvent.click(+ await timeline.findByRole("button", { name: /^Show tools:/ }),+ );+ for (const { name } of toolContentExamples) {+ await userEvent.click(+ timeline.getByRole("button", { name: `Expand ${name}` }),+ );+ }+ await expect(+ timeline.getByText("Monthly revenue increased."),+ ).toBeVisible();+ await expect(timeline.getByText(/monthly-revenue/)).toBeVisible();+ await expect(timeline.getByText(/chart-annotations/)).toBeVisible();+ await expect(+ timeline.getByText(/chart-annotations/).closest("pre"),+ ).toHaveTextContent("vendor-chart");+ await expect(+ timeline.getByText("Monthly revenue chart generated."),+ ).toBeVisible();+ await expect(timeline.getByText(/mixed-revenue/)).toBeVisible();+ await expect(+ timeline.getByText(/mixed-revenue/).closest("pre"),+ ).toHaveTextContent("points");+ const images = timeline.getAllByRole<HTMLImageElement>("img", {+ name: /Embedded image|revenue.svg/,+ });+ await expect(images).toHaveLength(4);+ for (const image of images) await expect(image).toBeVisible();+ await waitFor(async () => {+ for (const image of images) {+ await expect(image.naturalWidth).toBeGreaterThan(0);+ }+ });+ await expect(timeline.getByText(/chart-revenue/)).toBeVisible();+ await expect(+ timeline.getByText(/chart-revenue/).closest("pre"),+ ).toHaveTextContent("vendor-chart");+ await expect(timeline.queryByRole("note")).not.toBeInTheDocument();+ for (const label of [+ "revenue.svg",+ "image/svg+xml",+ "application/pdf",+ "File",+ ]) {+ await expect(timeline.getByText(label, { exact: true })).toBeVisible();+ }+ await expect(+ timeline.getByRole("link", { name: "/revenue-report.pdf" }),+ ).toHaveAttribute("href", "/revenue-report.pdf");+ const sidebar = within(canvas.getByRole("complementary"));+ await expect(+ sidebar.queryByRole("button", { name: "JSON message" }),+ ).not.toBeInTheDocument();+ },+});+export const ReasoningOnlyAndEmptySidebarStates = meta.story({name: "(Test) Reasoning-Only And Empty Sidebar States",args: {diff --git a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/components/SessionConversationTimeline/components/SessionConversationTimelineTrace/components/SessionTranscriptContent/SessionTranscriptContent.tsx b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/components/SessionConversationTimeline/components/SessionConversationTimelineTrace/components/SessionTranscriptContent/SessionTranscriptContent.tsxindex 05cfc99ecd..72dd1e5038 100644--- a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/components/SessionConversationTimeline/components/SessionConversationTimelineTrace/components/SessionTranscriptContent/SessionTranscriptContent.tsx+++ b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/components/SessionConversationTimeline/components/SessionConversationTimelineTrace/components/SessionTranscriptContent/SessionTranscriptContent.tsx@@ -1,4 +1,6 @@import { SessionTimelinePart } from "./components/SessionTimelinePart/SessionTimelinePart";+import { Codeblock } from "@/src/components/design-system/Codeblock/Codeblock";+import { assertUnreachable } from "@langfuse/shared";import { useState, type ComponentProps, type ReactNode } from "react";import { type TranscriptMessageGroup } from "@/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/fns/groupTranscriptMessages";import { getSessionTranscriptRows } from "@/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/fns/getSessionTranscriptRows";@@ -356,9 +358,33 @@ function SessionTranscriptTool({outputContent={row.result?.output.type === "content" &&row.result.output.value.length > 0- ? row.result.output.value.map((part, index) => (- <SessionTimelinePart key={index} part={part} />- ))+ ? row.result.output.value.map((part, index) => {+ if (part.type === "file") {+ return (+ <SessionTimelinePart+ key={index}+ part={part}+ mediaVariant="tool-output"+ />+ );+ }+ const value = (() => {+ if (part.type === "text") return part.text;+ if (part.type === "reasoning") return part.content;+ if (part.type === "data" || part.type === "custom")+ return part.value;+ return assertUnreachable(part);+ })();+ return (+ <div key={index} className="flex max-h-96 min-w-0 flex-col">+ <Codeblock+ label="Tool output"+ value={value}+ allowFormatting+ />+ </div>+ );+ }): undefined}{...status}diff --git a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/components/SessionConversationTimeline/components/SessionConversationTimelineTrace/components/SessionTranscriptContent/components/SessionTimelinePart/SessionTimelinePart.tsx b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/components/SessionConversationTimeline/components/SessionConversationTimelineTrace/components/SessionTranscriptContent/components/SessionTimelinePart/SessionTimelinePart.tsxindex 3a53f3aae3..97b7564389 100644--- a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/components/SessionConversationTimeline/components/SessionConversationTimelineTrace/components/SessionTranscriptContent/components/SessionTimelinePart/SessionTimelinePart.tsx+++ b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/components/SessionConversationalView/components/SessionConversationTimeline/components/SessionConversationTimelineTrace/components/SessionTranscriptContent/components/SessionTimelinePart/SessionTimelinePart.tsx@@ -1,5 +1,5 @@import { FileIcon, Wrench } from "lucide-react";-import { useState, type ReactNode } from "react";+import { useState, type ComponentProps, type ReactNode } from "react";import { SessionTimelineCollapsibleRow } from "../SessionTimelineCollapsibleRow/SessionTimelineCollapsibleRow";import { assertUnreachable } from "@langfuse/shared";import {@@ -23,9 +23,11 @@ import { decodeUnicodeEscapesOnly } from "@/src/utils/unicode";export function SessionTimelinePart({part,trailingContent,+ mediaVariant,}: {part: NormalizedMessage["parts"][number];trailingContent?: ReactNode;+ mediaVariant?: "tool-output";}) {if (part.type === "text") {if (part.refusal) {@@ -64,7 +66,7 @@ export function SessionTimelinePart({}if (part.type === "file") {- return <SessionTimelineFile part={part} />;+ return <SessionTimelineFile part={part} variant={mediaVariant} />;}if (part.type === "tool-call" || part.type === "tool-result") {@@ -170,24 +172,45 @@ function SessionTimelineReasoning({return assertUnreachable(content);}-function SessionTimelineFile({ part }: { part: FilePart }) {+function SessionTimelineFile({+ part,+ variant,+}: {+ part: FilePart;+ variant?: ComponentProps<typeof SessionTimelinePart>["mediaVariant"];+}) {const content = getSessionTimelineFileContent(part);+ const body = (+ <SessionTimelineFileBody+ content={content}+ filename={part.filename ?? "Embedded image"}+ />+ );return (<divclassName={cn(- "border-border/70 bg-background flex max-w-full flex-col gap-2 rounded-md border p-3",- content.kind === "fallback" ? "w-full" : "w-fit",+ variant === "tool-output"+ ? "bg-muted/30 flex w-full min-w-0 flex-col overflow-hidden rounded-md border"+ : "border-border/70 bg-background flex max-w-full flex-col gap-2 rounded-md border p-3",+ variant !== "tool-output" &&+ (content.kind === "fallback" ? "w-full" : "w-fit"),)}>- <div className="text-muted-foreground flex items-center gap-2 text-xs font-bold">+ <div+ className={cn(+ "text-muted-foreground flex items-center gap-2 text-xs",+ variant === "tool-output" ? "border-b px-2 py-1" : "font-bold",+ )}+ ><FileIcon className="icon-base" />{part.filename ?? part.mediaType ?? "File"}</div>- <SessionTimelineFileBody- content={content}- filename={part.filename ?? "Embedded image"}- />+ {variant === "tool-output" ? (+ <div className="min-w-0 p-3">{body}</div>+ ) : (+ body+ )}</div>);}diff --git a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/fns/groupTranscriptMessages.clienttest.ts b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/fns/groupTranscriptMessages.clienttest.tsindex 5c02121455..b0297cf33e 100644--- a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/fns/groupTranscriptMessages.clienttest.ts+++ b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/components/ConnectedSessionConversationTimeline/fns/groupTranscriptMessages.clienttest.ts@@ -31,6 +31,31 @@ const message = (parts: NormalizedMessagePart[]): NormalizedMessage => ({});describe("groupTranscriptMessages", () => {+ it("keeps a media-only result inside the tool row", () => {+ const toolCall = call("a");+ const toolResult: ToolResultPart = {+ ...result("a"),+ output: {+ type: "content",+ value: [+ {+ type: "file",+ content: { kind: "url", url: "https://example.com/a.png" },+ },+ {+ type: "file",+ content: { kind: "url", url: "https://example.com/b.png" },+ },+ ],+ },+ };+ const response = { ...message([toolResult]), role: "tool" as const };++ expect(groupTranscriptMessages([message([toolCall]), response])).toEqual([+ { type: "tool", message: response, call: toolCall, result: toolResult },+ ]);+ });+it("groups results at call positions while preserving surrounding parts and metadata", () => {const firstCall = call("a");const secondCall = call("b");diff --git a/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx b/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsxindex 80a6ea9266..c996cbc5fe 100644--- a/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx+++ b/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx@@ -73,9 +73,6 @@ it("bounds concurrent requests and releases slots after success and failure", as);const { result, unmount, client } = setup();await waitFor(() => expect(query).toHaveBeenCalledTimes(4));- expect(- query.mock.calls.every(([input]) => input.recoverToolResponses === true),- ).toBe(true);expect(query.mock.calls.every(([input]) => input.fallbackToRootIO === true),).toBe(true);diff --git a/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts b/web/src/features/sessions/hooks/useSessionTraceTranscripts.tsindex 6a7e551d2d..28aa426f38 100644--- a/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts+++ b/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts@@ -30,7 +30,6 @@ export function useSessionTraceTranscripts({projectId,traceId: trace.id,timestamp: trace.timestamp,- recoverToolResponses: true,fallbackToRootIO: true,};return {