🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(sessions): Improve tool call matching (#18478)
Ben Bachemab42ac03bd7f
diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 21723a2bc1..2b173d68ad 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -35,6 +35,14 @@"import": "./dist/src/server/transcript/ordering.js","require": "./dist/src/server/transcript/ordering.js"},+ "./src/server/transcript/threads": {+ "import": "./dist/src/server/transcript/threads.js",+ "require": "./dist/src/server/transcript/threads.js"+ },+ "./src/server/transcript/tool-calls": {+ "import": "./dist/src/server/transcript/tool-calls.js",+ "require": "./dist/src/server/transcript/tool-calls.js"+ },"./src/server/transcript/transcript": {"import": "./dist/src/server/transcript/transcript.js","require": "./dist/src/server/transcript/transcript.js"diff --git a/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts b/web/src/__tests__/server/unit/loadTraceTranscript.servertest.tsindex 77225e88a8..0e65bd49b5 100644--- a/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts+++ b/web/src/__tests__/server/unit/loadTraceTranscript.servertest.ts@@ -1,12 +1,25 @@import { beforeEach, expect, it, vi } from "vitest";import type { Transcript } from "@langfuse/shared/src/server";+import { assembleTranscript } from "@langfuse/shared/src/server/transcript/transcript";import { loadTraceTranscript } from "@/src/features/events/server/loadTraceTranscript";+import { normalizeSpanIO } from "@langfuse/shared/src/utils/normalized-io";+import type * as TranscriptThreads from "@langfuse/shared/src/server/transcript/threads";const mocks = vi.hoisted(() => ({assemble: vi.fn(),+ splitTurn: vi.fn(),+ splitTurnActual: vi.fn(),observations: vi.fn(),root: vi.fn(),}));+vi.mock(+ "@langfuse/shared/src/server/transcript/threads",+ async (importOriginal) => {+ const actual = await importOriginal<typeof TranscriptThreads>();+ mocks.splitTurnActual.mockImplementation(actual.splitTurn);+ return { ...actual, splitTurn: mocks.splitTurn };+ },+);vi.mock("@langfuse/shared/src/server", async () => {const { normalizeSpanIO } =await import("@langfuse/shared/src/utils/normalized-io");@@ -67,22 +80,301 @@ function fixture(],};mocks.assemble.mockReturnValue(transcript);+ mocks.splitTurn.mockReturnValue(transcript.threads[0]);mocks.observations.mockResolvedValue({- observations: responses.map((name, index) => ({- ...provenance,- id: `tool-${index}`,- type: "TOOL",- name,- parentObservationId: null,- input: null,- output: name,- metadata: {},- })),- totalCount: responses.length,+ observations: [+ {+ ...provenance,+ id: "generation",+ type: "GENERATION",+ input: [{ role: "user", content: "Run a tool" }],+ output: [+ {+ role: "assistant",+ parts: transcript.threads[0]!.currentTurn.messages[0]!.parts,+ },+ ],+ metadata: {},+ },+ ...responses.map((name, index) => ({+ ...provenance,+ id: `tool-${index}`,+ type: "TOOL",+ name,+ parentObservationId: null,+ input: null,+ output: name,+ metadata: {},+ })),+ ],+ totalCount: responses.length + 1,});return transcript;}-beforeEach(() => vi.resetAllMocks());+beforeEach(() => {+ vi.clearAllMocks();+ mocks.assemble.mockReset();+ mocks.observations.mockReset();+ mocks.root.mockReset();+ mocks.splitTurn.mockReset().mockImplementation(mocks.splitTurnActual);+});++it.each(["image", "media-token"])(+ "attaches reversed nested %s results by recovered call ID",+ async (scenario) => {+ const outer = {+ id: "outer",+ traceId: "trace",+ type: "GENERATION" as const,+ name: "generation",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: new Date(1),+ input: [{ role: "user", content: "Run tools" }],+ output: null,+ metadata: {},+ };+ const inner = {+ ...outer,+ id: "inner",+ parentObservationId: outer.id,+ startTime: new Date(1),+ input: [+ ...outer.input,+ {+ role: "assistant",+ content: [+ {+ type: "tool-call",+ toolCallId: "call-a",+ toolName: "search",+ input: { query: "outer" },+ },+ ],+ },+ ],+ };+ const tools = [inner, outer].map((parent, index) => ({+ ...outer,+ id: `tool-${parent.id}`,+ type: "TOOL" as const,+ name: "search",+ parentObservationId: parent.id,+ startTime: new Date(index + 2),+ input: { query: parent.id },+ output:+ scenario === "image"+ ? [+ {+ type: "image_url",+ image_url: { url: `https://example.com/${parent.id}.png` },+ },+ ]+ : `@@@langfuseMedia:type=image/jpeg|id=media-${parent.id}|source=base64@@@`,+ metadata: { callID: parent.id === "outer" ? "call-a" : "call-b" },+ }));+ const observations = [outer, inner, ...tools];+ const snapshot = structuredClone(observations);+ mocks.observations.mockResolvedValue({+ observations,+ totalCount: observations.length,+ });+ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ expect(result.transcript!.threads).toHaveLength(1);+ const messages = result.transcript!.threads[0]!.currentTurn.messages;+ for (const tool of tools) {+ const callIndex = messages.findIndex(+ (message) =>+ message.source === "output" &&+ message.parts.some(+ (part) =>+ part.type === "tool-call" &&+ part.toolCallId === tool.metadata.callID,+ ),+ );+ expect(callIndex).toBeGreaterThanOrEqual(0);+ const response = messages[callIndex + 1];+ expect(response).toMatchObject({ observationId: tool.id, role: "tool" });+ expect(response!.parts).toEqual(+ normalizeSpanIO({+ input: undefined,+ output: tool.output,+ metadata: {},+ }).messages.flatMap((message) => message.parts),+ );+ expect(response!.parts.some((part) => part.type === "file")).toBe(true);+ expect(response!.parts.some((part) => part.type === "tool-result")).toBe(+ false,+ );+ }+ expect(observations).toEqual(snapshot);+ },+);++it.each(["missing", "idless"])(+ "recovers a %s call despite duplicate observation rows",+ async (scenario) => {+ const input = { query: "search" };+ const generation = {+ id: "generation",+ traceId: "trace",+ type: "GENERATION" as const,+ name: "generation",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: new Date(1),+ input: [{ role: "user", content: "Run a tool" }],+ output:+ scenario === "idless"+ ? [+ {+ role: "assistant",+ parts: [{ type: "tool-call", toolName: "search", input }],+ },+ ]+ : null,+ metadata: {},+ };+ const tool = {+ ...generation,+ id: "tool",+ type: "TOOL" as const,+ name: "search",+ parentObservationId: generation.id,+ startTime: new Date(2),+ input,+ output: { answer: 42 },+ metadata: { callID: "call-1" },+ };+ const observations = [generation, { ...generation }, tool, { ...tool }];+ const snapshot = structuredClone(observations);+ mocks.observations.mockResolvedValue({+ observations,+ totalCount: observations.length,+ });+ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ const parts = result+ .transcript!.threads.flatMap((thread) => thread.currentTurn.messages)+ .flatMap((message) => message.parts);+ expect(parts.filter((part) => part.type === "tool-call")).toMatchObject([+ { toolCallId: "call-1" },+ ]);+ expect(parts.filter((part) => part.type === "tool-result")).toMatchObject([+ { toolCallId: "call-1", output: { answer: 42 } },+ ]);+ expect(observations).toEqual(snapshot);+ },+);++it.each(+ [+ {+ name: "image block",+ output: [+ {+ type: "image_url",+ image_url: { url: "https://example.com/image.png" },+ },+ ],+ },+ {+ name: "file block",+ output: [+ {+ type: "file",+ data: "aGVsbG8=",+ mediaType: "application/pdf",+ filename: "result.pdf",+ },+ ],+ },+ {+ name: "media token",+ output:+ "@@@langfuseMedia:type=image/jpeg|id=media-ref-image-1|source=base64@@@",+ },+ {+ name: "text with media token",+ output:+ "Screenshot: @@@langfuseMedia:type=image/jpeg|id=media-ref-image-1|source=base64@@@",+ },+ ].flatMap((scenario) =>+ [false, true].map((nativeCall) => ({ ...scenario, nativeCall })),+ ),+)(+ "preserves TOOL $name with native call=$nativeCall",+ async ({ output, nativeCall }) => {+ const input = { query: "screenshot" };+ const generation = {+ id: "generation",+ traceId: "trace",+ type: "GENERATION" as const,+ name: "generation",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: new Date(1),+ input: [{ role: "user", content: "Run a tool" }],+ output: nativeCall+ ? [+ {+ role: "assistant",+ parts: [+ {+ type: "tool-call",+ toolCallId: "call-1",+ toolName: "search",+ input,+ },+ ],+ },+ ]+ : null,+ metadata: {},+ };+ const tool = {+ ...generation,+ id: "tool",+ type: "TOOL" as const,+ name: "search",+ parentObservationId: generation.id,+ startTime: new Date(2),+ input,+ output,+ metadata: { callID: "call-1" },+ };+ const observations = [generation, tool];+ const snapshot = structuredClone(observations);+ const expectedParts = normalizeSpanIO({+ input: undefined,+ output,+ metadata: {},+ }).messages.flatMap((message) => message.parts);+ expect(expectedParts.some((part) => part.type === "file")).toBe(true);+ mocks.observations.mockResolvedValue({ observations, totalCount: 2 });+ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ const messages = result.transcript!.threads[0]!.currentTurn.messages;+ expect(+ messages+ .filter((message) => message.observationId === tool.id)+ .flatMap((message) => message.parts),+ ).toEqual(expectedParts);+ expect(+ messages+ .flatMap((message) => message.parts)+ .filter((part) => part.type === "tool-call"),+ ).toMatchObject([{ toolCallId: "call-1" }]);+ expect(observations).toEqual(snapshot);+ },+);it.each([{ parentObservationId: null, isRootObservation: true },@@ -152,6 +444,181 @@ it.each([},);+it.each([1, 2])(+ "does not complete ambiguous ID-less calls with %i TOOL executions",+ async (toolCount) => {+ const input = { code: "return 42" };+ const call = { type: "tool-call", toolName: "execute", input };+ const generation = {+ id: "generation",+ traceId: "trace",+ type: "GENERATION" as const,+ name: "generation",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: new Date(1),+ input: [{ role: "user", content: "Run a tool" }],+ output: [+ {+ role: "assistant",+ // Different raw encodings survive parser deduplication but normalize+ // to identical arguments, so neither call uniquely identifies the TOOL.+ parts:+ toolCount === 1+ ? [call, { ...call, input: JSON.stringify(input, null, 2) }]+ : [call],+ },+ ],+ metadata: {},+ };+ const observations = [+ generation,+ ...Array.from({ length: toolCount }, (_, index) => ({+ ...generation,+ id: `tool-${index}`,+ type: "TOOL" as const,+ name: "execute",+ parentObservationId: generation.id,+ startTime: new Date(index + 2),+ input,+ output: "42",+ metadata: { callID: `call-${index}` },+ })),+ ];+ mocks.observations.mockResolvedValue({+ observations,+ totalCount: observations.length,+ });+ mocks.assemble.mockImplementation(assembleTranscript);+ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ const calls = result+ .transcript!.threads[0]!.currentTurn.messages.flatMap(+ (message) => message.parts,+ )+ .filter((part) => part.type === "tool-call");+ const baseline = assembleTranscript([{ ...generation, nestingLevel: 0 }]);+ const originalCalls = baseline!.threads[0]!.currentTurn.messages.flatMap(+ (message) => message.parts,+ ).filter((part) => part.type === "tool-call");+ expect(calls).toEqual(originalCalls);+ expect(calls.every((part) => part.toolCallId === null)).toBe(true);+ },+);++it.each([+ { name: "JSON", output: { answer: 42 } },+ {+ name: "reasoning",+ output: [+ {+ role: "assistant",+ content: [+ { type: "reasoning", text: "Thinking", signature: "signature" },+ ],+ },+ ],+ },+])(+ "preserves generation $name while recovering calls through real assembly",+ async ({ output }) => {+ const generation = {+ id: "generation",+ traceId: "trace",+ type: "GENERATION" as const,+ name: "generation",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: new Date(1),+ input: [{ role: "user", content: "Run a tool" }],+ output,+ metadata: {},+ };+ const observations = [+ generation,+ {+ ...generation,+ id: "tool",+ type: "TOOL" as const,+ name: "execute",+ parentObservationId: generation.id,+ startTime: new Date(2),+ input: { code: "return 42" },+ output: "42",+ metadata: { callID: "call-1" },+ },+ ];+ const snapshot = structuredClone(observations);+ const baseline = assembleTranscript([{ ...generation, nestingLevel: 0 }]);+ mocks.observations.mockResolvedValue({ observations, totalCount: 2 });+ mocks.assemble.mockImplementation(assembleTranscript);++ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ const originalParts = baseline!.threads[0]!.currentTurn.messages.filter(+ (message) => message.source === "output",+ ).flatMap((message) => message.parts);+ const recoveredParts = result+ .transcript!.threads[0]!.currentTurn.messages.filter(+ (message) =>+ message.observationId === generation.id &&+ message.source === "output",+ )+ .flatMap((message) => message.parts);+ expect(recoveredParts.filter((part) => part.type !== "tool-call")).toEqual(+ originalParts,+ );+ expect(+ recoveredParts.filter((part) => part.type === "tool-call"),+ ).toMatchObject([{ toolCallId: "call-1" }]);+ expect(observations).toEqual(snapshot);+ },+);++it.each([+ { structureCount: 10000, contentCount: 1000, cutoff: false },+ { structureCount: 10001, contentCount: 1000, cutoff: true },+ { structureCount: 10000, contentCount: 1001, cutoff: true },+])(+ "merges capped I/O with structure and reports cutoff: %j",+ async ({ structureCount, contentCount, cutoff }) => {+ fixture();+ const span = { id: "span", type: "SPAN" };+ const generation = { id: "generation", type: "GENERATION" };+ const tool = { id: "tool", type: "TOOL" };+ const generationWithContent = {+ ...generation,+ input: "question",+ output: "answer",+ };+ mocks.observations+ .mockResolvedValueOnce({+ observations: [span, generation, tool],+ totalCount: structureCount,+ })+ .mockResolvedValueOnce({+ observations: [generationWithContent],+ totalCount: contentCount,+ });++ const result = await loadTraceTranscript(trace);++ expect(mocks.assemble).toHaveBeenCalledWith([span, generationWithContent]);+ expect(result.cutoff).toBe(cutoff);+ expect(mocks.observations).toHaveBeenNthCalledWith(1, trace);+ expect(mocks.observations).toHaveBeenNthCalledWith(2, {+ ...trace,+ selectIOAndMetadata: true,+ types: ["GENERATION", "TOOL"],+ limit: 1000,+ });+ },+);+it("falls back to seeded JSON chat messages with an empty-string root parent", async () => {const root = {id: "t-seed-media-image-only",@@ -245,12 +712,239 @@ it("leaves the transcript empty when the root has no input/output", async () =>.transcript,).toBeNull();});+it("leaves mixed text/data responses unchanged", async () => {+ const original = fixture(["search"], ["search"]);+ original.threads[0]!.currentTurn.messages[1]!.parts = [+ { type: "text", text: "result" },+ { type: "data", value: { answer: 42 } },+ ];+ const snapshot = structuredClone(original);++ expect(+ (await loadTraceTranscript({ ...trace, recoverToolResponses: true }))+ .transcript,+ ).toEqual(snapshot);+ expect(original).toEqual(snapshot);+});++it("preserves explicit results that conflict with TOOL metadata", async () => {+ const observations = [+ {+ id: "generation",+ traceId: "trace",+ type: "GENERATION",+ name: "generation",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: new Date(1),+ input: null,+ output: [+ {+ role: "assistant",+ parts: [+ {+ type: "tool-call",+ toolCallId: "explicit",+ toolName: "search",+ input: {},+ },+ ],+ },+ ],+ metadata: {},+ },+ {+ id: "tool",+ traceId: "trace",+ type: "TOOL",+ name: "search",+ parentObservationId: "generation",+ startTime: new Date(2),+ endTime: new Date(3),+ input: {},+ output: { role: "tool", tool_call_id: "explicit", content: "result" },+ metadata: { callID: "conflicting" },+ },+ ];+ const snapshot = structuredClone(observations);+ mocks.observations.mockResolvedValue({ observations, totalCount: 2 });+ mocks.assemble.mockImplementation(assembleTranscript);++ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });++ const parts = result.transcript!.threads[0]!.currentTurn.messages.flatMap(+ (message) => message.parts,+ );+ expect(parts.filter((part) => part.type === "tool-result")).toMatchObject([+ { toolCallId: "explicit", output: "result" },+ ]);+ expect(+ parts.some(+ (part) => part.type === "tool-call" && part.toolCallId === "conflicting",+ ),+ ).toBe(false);+ expect(observations).toEqual(snapshot);+});++it.each([+ "missing-generation-call",+ "existing-generation-call",+ "idless-generation-call",+ "existing-tool-result",+])(+ "pairs TOOL metadata call IDs through real assembly: %s",+ async (scenario) => {+ const input = { code: "return 42" };+ const output = { title: "Result", output: "42" };+ const observations = [+ {+ id: "generation",+ traceId: "trace",+ type: "GENERATION" as const,+ name: "generation",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: new Date(1),+ input: [{ role: "user", content: "Run a tool" }],+ output:+ scenario === "existing-generation-call" ||+ scenario === "idless-generation-call"+ ? [+ {+ role: "assistant",+ parts: [+ {+ type: "tool-call",+ toolCallId:+ scenario === "idless-generation-call" ? null : "call-1",+ toolName: "execute",+ input,+ },+ ],+ },+ ]+ : [{ role: "assistant", content: "" }],+ metadata: {},+ },+ {+ id: "tool",+ traceId: "trace",+ type: "TOOL" as const,+ name: "execute",+ parentObservationId: "generation",+ startTime: new Date(2),+ endTime: new Date(3),+ input,+ output:+ scenario === "existing-tool-result"+ ? {+ role: "tool",+ name: "execute",+ tool_call_id: "call-1",+ content: output,+ }+ : output,+ metadata: { callID: "call-1" },+ },+ ];+ const snapshot = structuredClone(observations);+ mocks.assemble.mockImplementation(assembleTranscript);+ mocks.observations.mockResolvedValue({+ observations,+ totalCount: observations.length,+ });+ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ const messages = result.transcript!.threads[0]!.currentTurn.messages;+ expect(+ messages+ .flatMap((message) => message.parts)+ .filter((part) => part.type === "tool-call"),+ ).toMatchObject([+ { type: "tool-call", toolCallId: "call-1", toolName: "execute", input },+ ]);+ expect(+ messages.find((message) => message.observationId === "tool"),+ ).toMatchObject({+ role: "tool",+ source: "output",+ observationId: "tool",+ startTime: new Date(2),+ endTime: new Date(3),+ parts: [+ {+ type: "tool-result",+ toolCallId: "call-1",+ toolName: "execute",+ output,+ },+ ],+ });+ expect(observations).toEqual(snapshot);+ expect((await loadTraceTranscript(trace)).transcript).toEqual(+ assembleTranscript(+ observations.map((observation) => ({+ ...observation,+ nestingLevel: 0,+ })),+ ),+ );+ },+);++it.each(["duplicate-call-id", "different-trace", "non-generation-parent"])(+ "does not reconstruct ambiguous TOOL metadata: %s",+ async (scenario) => {+ const original = fixture(["execute"], ["execute"]);+ const generation = {+ id: "generation",+ traceId: "trace",+ type: "GENERATION",+ name: "generation",+ parentObservationId: null,+ startTime: new Date(0),+ endTime: null,+ input: null,+ output: null,+ metadata: {},+ };+ const tool = {+ ...generation,+ id: "tool",+ type: "TOOL",+ name: "execute",+ parentObservationId: "generation",+ metadata: { callID: "call-1" },+ input: { code: "return 42" },+ output: { answer: 42 },+ };+ if (scenario === "different-trace") generation.traceId = "other-trace";+ if (scenario === "non-generation-parent") generation.type = "SPAN";+ const observations =+ scenario === "duplicate-call-id"+ ? [generation, tool, { ...tool, id: "tool-duplicate" }]+ : [generation, tool];+ mocks.observations.mockResolvedValue({+ observations,+ totalCount: observations.length,+ });+ await loadTraceTranscript({ ...trace, recoverToolResponses: true });+ expect(original.threads[0]!.currentTurn.messages[1]!.parts[0]?.type).toBe(+ "text",+ );+ },+);it("opts into provenance-based reverse multi-call responses without mutating assembly", async () => {const original = fixture();const snapshot = structuredClone(original);const result = await loadTraceTranscript({...trace,- pairTextToolResponses: true,+ recoverToolResponses: true,});expect(result.transcript?.threads[0]?.currentTurn.messages@@ -288,6 +982,102 @@ it("leaves the default transcript unchanged", async () => {const original = fixture();expect((await loadTraceTranscript(trace)).transcript).toBe(original);});+it.each([+ { output: { answer: 42 } },+ { output: [{ answer: 42 }] },+ { output: null },+])(+ "pairs JSON-only TOOL output using observation provenance: %j",+ async ({ output }) => {+ const original = fixture(["search"], ["search"]);+ original.threads[0]!.currentTurn.messages[1]!.parts = [+ { type: "data", value: output },+ ];+ const snapshot = structuredClone(original);+ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ expect(+ result.transcript?.threads[0]?.currentTurn.messages[1]?.parts,+ ).toEqual([+ { type: "tool-result", toolCallId: "call-0", toolName: "search", output },+ ]);+ expect(original).toEqual(snapshot);+ expect((await loadTraceTranscript(trace)).transcript).toBe(original);+ },+);++it("preserves multiple JSON parts as an output array", async () => {+ const original = fixture(["search"], ["search"]);+ original.threads[0]!.currentTurn.messages[1]!.parts = [+ { type: "data", value: { answer: 42 } },+ { type: "data", value: { confidence: 0.9 } },+ ];+ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ expect(result.transcript?.threads[0]?.currentTurn.messages[1]?.parts).toEqual(+ [+ {+ type: "tool-result",+ toolCallId: "call-0",+ toolName: "search",+ output: [{ answer: 42 }, { confidence: 0.9 }],+ },+ ],+ );+});++it.each([+ "input",+ "wrong-observation-type",+ "ambiguous-call",+ "explicit-result",+])("does not infer JSON tool results for %s", async (scenario) => {+ const original = fixture(["search"], ["search"]);+ const messages = original.threads[0]!.currentTurn.messages;+ messages[1]!.parts = [{ type: "data", value: { answer: 42 } }];+ if (scenario === "input") messages[1]!.source = "input";+ if (scenario === "wrong-observation-type") {+ mocks.observations.mockResolvedValue({+ observations: [+ {+ id: "tool-0",+ traceId: "trace",+ type: "GENERATION",+ name: "search",+ input: [{ role: "user", content: "Run a tool" }],+ },+ ],+ totalCount: 1,+ });+ }+ if (scenario === "ambiguous-call") {+ messages[0]!.parts.push({+ type: "tool-call",+ toolCallId: "other",+ toolName: "search",+ input: {},+ });+ }+ if (scenario === "explicit-result") {+ messages.push({+ ...messages[1]!,+ parts: [+ { type: "tool-result", toolCallId: "call-0", output: "explicit" },+ ],+ });+ }+ const snapshot = structuredClone(original);+ const result = await loadTraceTranscript({+ ...trace,+ recoverToolResponses: true,+ });+ expect(result.transcript).toEqual(snapshot);+ expect(original).toEqual(snapshot);+});it.each([{ names: ["search", "search"], responses: ["search"] },{ names: ["search"], responses: ["search", "search"] },@@ -296,7 +1086,7 @@ it.each([async ({ names, responses }) => {const original = fixture(names, responses);expect(- (await loadTraceTranscript({ ...trace, pairTextToolResponses: true }))+ (await loadTraceTranscript({ ...trace, recoverToolResponses: true })).transcript,).toEqual(original);},@@ -308,7 +1098,7 @@ it("preserves explicit result precedence", async () => {parts: [{ type: "tool-result", toolCallId: "call-0", output: "explicit" }],});expect(- (await loadTraceTranscript({ ...trace, pairTextToolResponses: true }))+ (await loadTraceTranscript({ ...trace, recoverToolResponses: true })).transcript,).toEqual(original);});@@ -316,7 +1106,7 @@ it("requires exact response trace provenance even for adjacent single calls", asconst original = fixture(["search"], ["search"]);original.threads[0]!.currentTurn.messages[1]!.traceId = "other-trace";expect(- (await loadTraceTranscript({ ...trace, pairTextToolResponses: true }))+ (await loadTraceTranscript({ ...trace, recoverToolResponses: true })).transcript,).toEqual(original);});@@ -334,7 +1124,7 @@ it.each(["missing-observation", "conflicting-name", "reused-id"])(if (scenario === "reused-id")messages[0]!.parts.push({ ...messages[0]!.parts[0]! });expect(- (await loadTraceTranscript({ ...trace, pairTextToolResponses: true }))+ (await loadTraceTranscript({ ...trace, recoverToolResponses: true })).transcript,).toEqual(original);},@@ -345,7 +1135,7 @@ it("leaves history untouched when it shares current-turn message references", asoriginal.threads[0]!.currentTurn.messages;const result = await loadTraceTranscript({...trace,- pairTextToolResponses: true,+ recoverToolResponses: true,});expect(result.transcript?.threads[0]?.conversationHistory[1]?.parts[0]?.type,diff --git a/web/src/features/events/server/eventsRouter.ts b/web/src/features/events/server/eventsRouter.tsindex 15cd1be148..ea452a74df 100644--- a/web/src/features/events/server/eventsRouter.ts+++ b/web/src/features/events/server/eventsRouter.ts@@ -445,7 +445,7 @@ export const eventsRouter = createTRPCRouter({projectId: zodSchema.string(),traceId: zodSchema.string(),timestamp: zodSchema.date().optional(),- pairTextToolResponses: zodSchema.boolean().default(false),+ recoverToolResponses: zodSchema.boolean().default(false),fallbackToRootIO: zodSchema.boolean().default(false),}),)@@ -494,7 +494,7 @@ export const eventsRouter = createTRPCRouter({projectId: input.projectId,traceId: input.traceId,timestamp,- pairTextToolResponses: input.pairTextToolResponses,+ recoverToolResponses: input.recoverToolResponses,fallbackToRootIO: input.fallbackToRootIO,});},diff --git a/web/src/features/events/server/loadTraceTranscript.ts b/web/src/features/events/server/loadTraceTranscript.tsindex 78f3322c7f..cf341b3b99 100644--- a/web/src/features/events/server/loadTraceTranscript.ts+++ b/web/src/features/events/server/loadTraceTranscript.ts@@ -1,3 +1,4 @@+import { isEqual, partition } from "lodash";import {assembleTranscript,getObservationByIdFromEventsTable,@@ -7,13 +8,30 @@ import {normalizeIO,type Transcript,} from "@langfuse/shared/src/server";-import type { ToolCallPart } from "@langfuse/shared/src/utils/normalized-io";+import {+ normalizeSpanIO,+ type NormalizedMessage,+ type ToolCallPart,+} from "@langfuse/shared/src/utils/normalized-io";+import {+ append,+ createThread,+ findThread,+ messageKey,+ splitTurn,+ type ThreadState,+} from "@langfuse/shared/src/server/transcript/threads";+import { createToolCallRegistry } from "@langfuse/shared/src/server/transcript/tool-calls";/** Generations and tools whose I/O one transcript reads at most. */const MAX_TRANSCRIPT_OBSERVATIONS = 1_000;const TRANSCRIPT_OBSERVATION_TYPES = new Set(["GENERATION", "TOOL"]);+type Observation = Awaited<+ ReturnType<typeof getObservationsForTraceFromEventsTable>+>["observations"][number];+/*** Load a trace's observations from the events table, walk them in trace tree* order, and assemble the transcript of its generations and tools.@@ -24,11 +42,11 @@ export async function loadTraceTranscript(trace: {/** Trace timestamp; observations from one hour before it onwards are read. */timestamp: Date;/**- * For session rendering, convert text-only TOOL responses into ID-bearing- * tool results when observation names uniquely identify their calls.+ * For session rendering, recover calls from TOOL metadata and pair text-only+ * or JSON-only responses when observation names uniquely identify their calls.* Ambiguous responses remain unchanged. Disabled by default.*/- pairTextToolResponses?: boolean;+ recoverToolResponses?: boolean;/** Use root observation I/O when no generation/tool transcript exists. */fallbackToRootIO?: boolean;}): Promise<{@@ -55,7 +73,20 @@ export async function loadTraceTranscript(trace: {if (withContent) return [withContent];return TRANSCRIPT_OBSERVATION_TYPES.has(o.type) ? [] : [o];});- let transcript = assembleTranscript(orderObservations(observations));+ let transcript = (() => {+ if (!trace.recoverToolResponses) {+ return assembleTranscript(orderObservations(observations));+ }+ const recovered = recoverToolCalls(observations);+ return pairToolResponses(+ assembleSessionTraceTranscript(+ orderObservations(recovered.observations),+ recovered.transformOutput,+ recovered.mediaCallIds,+ ),+ recovered.observationsByTraceAndId,+ );+ })();if (!transcript && trace.fallbackToRootIO) {const root = structure.observations.find((observation) =>@@ -102,113 +133,405 @@ export async function loadTraceTranscript(trace: {}}}- // TODO: Evaluate moving general pairing into shared transcript assembly once- // it preserves resolved call identity for text-only tool responses.- if (trace.pairTextToolResponses && transcript) {- const observationsByTraceAndId = new Map<- (typeof observations)[number]["traceId"],- Map<string, (typeof observations)[number]>- >();- for (const observation of observations) {- let byId = observationsByTraceAndId.get(observation.traceId);- if (!byId) {- byId = new Map();- observationsByTraceAndId.set(observation.traceId, byId);+ const cutoff =+ structure.totalCount > MAX_OBSERVATIONS_PER_TRACE ||+ content.totalCount > MAX_TRANSCRIPT_OBSERVATIONS;+ return {+ transcript,+ cutoff,+ };+}++// TODO: Look into moving session tool recovery into shared transcript assembly+// so normalized-output recovery does not need separate assembly orchestration.+function assembleSessionTraceTranscript(+ observations: Array<Observation & { nestingLevel: number }>,+ transformOutput: (+ observation: Observation,+ messages: NormalizedMessage[],+ ) => NormalizedMessage[],+ mediaCallIds: Map<string, string>,+): Transcript | null {+ const threads: ThreadState[] = [];+ const toolCalls = createToolCallRegistry();+ for (const observation of observations.filter(+ (+ observation,+ ): observation is Observation & { nestingLevel: number; traceId: string } =>+ (observation.type === "GENERATION" || observation.type === "TOOL") &&+ observation.traceId !== null,+ )) {+ const { messages } = normalizeSpanIO({+ input: observation.type === "TOOL" ? undefined : observation.input,+ output: observation.output,+ metadata: observation.metadata,+ });+ const [inputMessages, outputMessages] = partition(+ messages,+ (message) => message.source === "input",+ );+ const [input, output] = partition(+ [...inputMessages, ...transformOutput(observation, outputMessages)].map(+ (message) => ({ message, key: messageKey(message) }),+ ),+ ({ message }) => message.source === "input",+ );+ if (observation.type === "TOOL") {+ const callId = mediaCallIds.get(+ JSON.stringify([observation.traceId, observation.id]),+ );+ if (callId) {+ const marker: NormalizedMessage = {+ role: "tool",+ source: "output",+ parts: [{ type: "tool-result", toolCallId: callId, output: null }],+ };+ // Let the registry resolve identity and placement, then retain the+ // original media parts instead of exposing a JSON-only result marker.+ toolCalls.attachToolOutput(observation, [+ { message: marker, key: messageKey(marker) },+ ]);+ for (const thread of threads) {+ const response = thread.messages.find(+ (message) =>+ message.observationId === observation.id &&+ message.traceId === observation.traceId &&+ message.parts.some(+ (part) =>+ part.type === "tool-result" && part.toolCallId === callId,+ ),+ );+ if (response) {+ response.parts = output.flatMap(({ message }) => message.parts);+ break;+ }+ }+ continue;}- // Preserve the first match, as the previous array lookup did.- if (!byId.has(observation.id)) byId.set(observation.id, observation);+ toolCalls.attachToolOutput(observation, output);+ continue;}- transcript = {- ...transcript,- threads: transcript.threads.map((thread) => {- const messages = thread.currentTurn.messages;- const calls = messages.flatMap((message) =>- message.parts.filter(- (part): part is ToolCallPart => part.type === "tool-call",- ),+ if (input.length === 0 && output.length === 0) continue;+ let thread = findThread(threads, input);+ if (!thread) {+ thread = createThread();+ threads.push(thread);+ }+ append(thread, observation, input, output, toolCalls);+ }+ return threads.length ? { threads: threads.map(splitTurn) } : null;+}++function recoverToolCalls(observations: Observation[]) {+ const observationsByTraceAndId = new Map<+ Observation["traceId"],+ Map<string, Observation>+ >();+ const callsByParent = new Map<string, ToolCallPart[]>();+ const uniqueObservations: Observation[] = [];+ const toolsByObservation = new Map<string, string>();+ const callCountsByTrace = new Map<+ Observation["traceId"],+ Map<string, number>+ >();+ for (const observation of observations) {+ let byId = observationsByTraceAndId.get(observation.traceId);+ if (!byId) {+ byId = new Map();+ observationsByTraceAndId.set(observation.traceId, byId);+ }+ // Preserve the first match, as the previous array lookup did.+ if (byId.has(observation.id)) continue;+ byId.set(observation.id, observation);+ uniqueObservations.push(observation);++ if (observation.type !== "TOOL") continue;+ const callId = observation.metadata?.callID;+ if (typeof callId !== "string" || !callId) continue;+ let counts = callCountsByTrace.get(observation.traceId);+ if (!counts) {+ counts = new Map();+ callCountsByTrace.set(observation.traceId, counts);+ }+ counts.set(callId, (counts.get(callId) ?? 0) + 1);+ }++ // Some plugins omit calls from generation output. A TOOL's unique call ID+ // and same-trace generation parent recover that anchor without name guessing.+ for (const observation of uniqueObservations) {+ if (+ observation.type !== "TOOL" ||+ !observation.name ||+ !observation.parentObservationId+ )+ continue;+ const callId = observation.metadata?.callID;+ if (typeof callId !== "string" || !callId) continue;+ if (callCountsByTrace.get(observation.traceId)?.get(callId) !== 1) continue;+ const parent = observationsByTraceAndId+ .get(observation.traceId)+ ?.get(observation.parentObservationId);+ if (parent?.type !== "GENERATION") continue;+ const outputParts = normalizeSpanIO({+ input: undefined,+ output: observation.output,+ metadata: undefined,+ }).messages.flatMap((message) => message.parts);+ const explicitResults = outputParts.flatMap((part) =>+ part.type === "tool-result" && part.toolCallId ? [part] : [],+ );+ if (explicitResults.some((part) => part.toolCallId !== callId)) continue;+ const normalized = normalizeSpanIO({+ input: undefined,+ metadata: undefined,+ output: {+ type: "tool-call",+ toolCallId: callId,+ toolName: observation.name,+ input: observation.input,+ },+ });+ const call = normalized.messages+ .flatMap((message) => message.parts)+ .find((part): part is ToolCallPart => part.type === "tool-call");+ if (!call) continue;+ const parentKey = JSON.stringify([parent.traceId, parent.id]);+ const calls = callsByParent.get(parentKey);+ if (calls) calls.push(call);+ else callsByParent.set(parentKey, [call]);+ if (!explicitResults.length) {+ toolsByObservation.set(+ JSON.stringify([observation.traceId, observation.id]),+ callId,+ );+ }+ }++ const resolvedCallIds = new Set<string>();+ const mediaCallIds = new Map<string, string>();+ const transformOutput = (+ observation: Observation,+ messages: NormalizedMessage[],+ ): NormalizedMessage[] => {+ if (observation.type === "TOOL") {+ const callId = toolsByObservation.get(+ JSON.stringify([observation.traceId, observation.id]),+ );+ if (+ !callId ||+ !resolvedCallIds.has(JSON.stringify([observation.traceId, callId]))+ )+ return messages;+ const parts = messages.flatMap((message) => message.parts);+ // Tool-result payloads are JSON; wrapping media would discard the file+ // parts the session renderer needs for previews.+ if (!parts.length) return messages;+ if (+ !parts.every((part) => part.type === "text") &&+ !parts.every((part) => part.type === "data")+ ) {+ mediaCallIds.set(+ JSON.stringify([observation.traceId, observation.id]),+ callId,);- const explicitIds = new Set(- messages.flatMap((message) =>- message.parts.flatMap((part) =>- part.type === "tool-result" && part.toolCallId- ? [part.toolCallId]- : [],- ),- ),+ return messages;+ }+ return normalizeSpanIO({+ input: undefined,+ metadata: undefined,+ output: {+ role: "tool",+ name: observation.name,+ tool_call_id: callId,+ content: observation.output,+ },+ }).messages;+ }+ const calls = callsByParent.get(+ JSON.stringify([observation.traceId, observation.id]),+ );+ if (!calls || observation.type !== "GENERATION") return messages;+ const existingIds = new Set(+ messages.flatMap((message) =>+ message.parts.flatMap((part) =>+ part.type === "tool-call" ? [part.toolCallId] : [],+ ),+ ),+ );+ const idlessCalls = messages.flatMap((message) =>+ message.parts.filter(+ (part): part is ToolCallPart =>+ part.type === "tool-call" && !part.toolCallId,+ ),+ );+ const replacements = new Map<ToolCallPart, ToolCallPart>();+ const missingCalls: ToolCallPart[] = [];+ for (const call of calls) {+ if (existingIds.has(call.toolCallId)) {+ resolvedCallIds.add(+ JSON.stringify([observation.traceId, call.toolCallId]),);- const candidates = new Map<number, ToolCallPart>();- let anchor: (typeof messages)[number] | undefined;- for (const [index, message] of messages.entries()) {- if (message.role !== "tool") {- anchor =- message.role === "assistant" && message.source === "output"- ? message- : undefined;- continue;- }- if (- message.source !== "output" ||- !message.parts.length ||- !message.parts.every((part) => part.type === "text")- )- continue;- const observation = observationsByTraceAndId- .get(message.traceId)- ?.get(message.observationId);- // Provenance must identify a TOOL; positional guessing cannot override it.- if (observation?.type !== "TOOL" || !observation.name || !anchor)- continue;- const matches = anchor.parts.filter(- (part): part is ToolCallPart =>- part.type === "tool-call" && part.toolName === observation.name,+ continue;+ }+ const matches = idlessCalls.filter(+ (existing) =>+ existing.toolName === call.toolName &&+ isEqual(existing.input, call.input),+ );+ // Ambiguous existing calls cannot safely be completed or duplicated.+ if (matches.length > 0) {+ const recoveredMatches = calls.filter(+ (candidate) =>+ candidate.toolName === call.toolName &&+ isEqual(candidate.input, call.input),+ );+ if (matches.length === 1 && recoveredMatches.length === 1) {+ replacements.set(matches[0]!, call);+ resolvedCallIds.add(+ JSON.stringify([observation.traceId, call.toolCallId]),);- if (matches.length !== 1) continue;- const call = matches[0]!;- if (- !call.toolCallId ||- explicitIds.has(call.toolCallId) ||- calls.filter((other) => other.toolCallId === call.toolCallId)- .length !== 1- )- continue;- candidates.set(index, call);}- return {- ...thread,- currentTurn: {- ...thread.currentTurn,- messages: messages.map((message, index) => {- const call = candidates.get(index);- if (- !call ||- [...candidates.values()].filter(- (other) => other.toolCallId === call.toolCallId,- ).length !== 1- )- return message;- return {- ...message,- parts: [- {- type: "tool-result" as const,- toolCallId: call.toolCallId,- toolName: call.toolName,- output: message.parts- .map((part) => (part.type === "text" ? part.text : ""))- .join("\n"),- },- ],- };- }),- },- };+ continue;+ }+ missingCalls.push(call);+ resolvedCallIds.add(+ JSON.stringify([observation.traceId, call.toolCallId]),+ );+ }+ if (!replacements.size && !missingCalls.length) return messages;+ const recoveredMessages = messages.map((message) => ({+ ...message,+ parts: message.parts.map((part) => {+ if (part.type !== "tool-call") return part;+ const replacement = replacements.get(part);+ return replacement+ ? { ...part, toolCallId: replacement.toolCallId }+ : part;}),- };- }+ }));+ return missingCalls.length+ ? [+ ...recoveredMessages,+ { role: "assistant", source: "output", parts: missingCalls },+ ]+ : recoveredMessages;+ };return {- transcript,- cutoff:- structure.totalCount > MAX_OBSERVATIONS_PER_TRACE ||- content.totalCount > MAX_TRANSCRIPT_OBSERVATIONS,+ observations: uniqueObservations,+ observationsByTraceAndId,+ transformOutput,+ mediaCallIds,+ };+}++function pairToolResponses(+ transcript: Transcript | null,+ observationsByTraceAndId: Map<+ Observation["traceId"],+ Map<string, Observation>+ >,+): Transcript | null {+ // TODO: Evaluate moving general pairing into shared transcript assembly once+ // it preserves resolved call identity for text-only and JSON-only tool responses.+ if (!transcript) return null;+ return {+ ...transcript,+ threads: transcript.threads.map((thread) => {+ // Historical messages are already established context, not recovery targets.+ const messages = thread.currentTurn.messages;+ const callCounts = new Map<string, number>();+ for (const message of messages) {+ for (const part of message.parts) {+ if (part.type !== "tool-call" || !part.toolCallId) continue;+ callCounts.set(+ part.toolCallId,+ (callCounts.get(part.toolCallId) ?? 0) + 1,+ );+ }+ }+ const explicitIds = new Set(+ messages.flatMap((message) =>+ message.parts.flatMap((part) =>+ part.type === "tool-result" && part.toolCallId+ ? [part.toolCallId]+ : [],+ ),+ ),+ );+ const candidates = new Map<number, ToolCallPart>();+ const candidateCounts = new Map<string, number>();+ const anchorCallsByName = new Map<string, ToolCallPart | null>();+ for (const [index, message] of messages.entries()) {+ if (message.role !== "tool") {+ // Name matching is limited to the immediately preceding assistant+ // output and its consecutive tool responses.+ anchorCallsByName.clear();+ if (message.role === "assistant" && message.source === "output") {+ for (const part of message.parts) {+ if (part.type !== "tool-call") continue;+ // A repeated name cannot identify one call unambiguously.+ anchorCallsByName.set(+ part.toolName,+ anchorCallsByName.has(part.toolName) ? null : part,+ );+ }+ }+ continue;+ }+ if (+ message.source !== "output" ||+ !message.parts.length ||+ (!message.parts.every((part) => part.type === "text") &&+ !message.parts.every((part) => part.type === "data"))+ )+ continue;+ const observation = observationsByTraceAndId+ .get(message.traceId)+ ?.get(message.observationId);+ // Provenance must identify a TOOL; positional guessing cannot override it.+ if (observation?.type !== "TOOL" || !observation.name) continue;+ const call = anchorCallsByName.get(observation.name);+ if (!call) continue;+ if (!call.toolCallId) continue;+ if (explicitIds.has(call.toolCallId)) continue;+ if (callCounts.get(call.toolCallId) !== 1) continue;+ candidates.set(index, call);+ candidateCounts.set(+ call.toolCallId,+ (candidateCounts.get(call.toolCallId) ?? 0) + 1,+ );+ }+ return {+ ...thread,+ currentTurn: {+ ...thread.currentTurn,+ messages: messages.map((message, index) => {+ const call = candidates.get(index);+ if (!call?.toolCallId) return message;+ if (candidateCounts.get(call.toolCallId) !== 1) return message;+ const output = (() => {+ if (message.parts.every((part) => part.type === "text")) {+ return message.parts.map((part) => part.text).join("\n");+ }+ const values = message.parts.flatMap((part) =>+ part.type === "data" ? [part.value] : [],+ );+ return values.length === 1 ? values[0] : values;+ })();+ return {+ ...message,+ parts: [+ {+ type: "tool-result" as const,+ toolCallId: call.toolCallId,+ toolName: call.toolName,+ output,+ },+ ],+ };+ }),+ },+ };+ }),};}diff --git a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsxindex 21ddee6ee6..3393ed1034 100644--- a/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx+++ b/web/src/features/sessions/components/ConnectedModernSessionBodyTimeline/ConnectedModernSessionBodyTimeline.tsx@@ -108,7 +108,7 @@ export function ConnectedModernSessionBodyTimeline({projectId,traceId: trace.id,timestamp: trace.timestamp,- pairTextToolResponses: true,+ recoverToolResponses: true,}) !== undefined,);const searchQuery = allTranscriptsLoadeddiff --git a/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx b/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsxindex f4ed15cf2b..80a6ea9266 100644--- a/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx+++ b/web/src/features/sessions/hooks/useSessionTraceTranscripts.clienttest.tsx@@ -74,7 +74,7 @@ it("bounds concurrent requests and releases slots after success and failure", asconst { result, unmount, client } = setup();await waitFor(() => expect(query).toHaveBeenCalledTimes(4));expect(- query.mock.calls.every(([input]) => input.pairTextToolResponses === true),+ query.mock.calls.every(([input]) => input.recoverToolResponses === true),).toBe(true);expect(query.mock.calls.every(([input]) => input.fallbackToRootIO === true),diff --git a/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts b/web/src/features/sessions/hooks/useSessionTraceTranscripts.tsindex 210562e069..6a7e551d2d 100644--- a/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts+++ b/web/src/features/sessions/hooks/useSessionTraceTranscripts.ts@@ -30,7 +30,7 @@ export function useSessionTraceTranscripts({projectId,traceId: trace.id,timestamp: trace.timestamp,- pairTextToolResponses: true,+ recoverToolResponses: true,fallbackToRootIO: true,};return {