🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): stop silencing MCP tool failures into the sandbox (#16082)
Lysander Kiesel62033434be81
diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.tsindex 87c16dfcab..bf1ef0a3bd 100644--- a/packages/shared/src/in-app-agent/server/persistence.ts+++ b/packages/shared/src/in-app-agent/server/persistence.ts@@ -43,6 +43,7 @@ import {getSandboxInAppAgentMcpToolResultContent,IN_APP_AGENT_SANDBOX_TOOL_NAMES,} from "./tools";+import { getToolFailureMessage } from "./toolErrors";export const ACTIVE_RUN_CONFLICT_MESSAGE ="Assistant is already responding in this conversation";@@ -315,6 +316,14 @@ export function createSandboxToolCallFileAccumulator(const draft = drafts.get(toolCall.toolCallId);drafts.delete(toolCall.toolCallId);completedToolCallIds.add(toolCall.toolCallId);++ // The MCP wrapper already classified the failure onto `error`, so failures+ // never become sandbox tool_calls files. Marking the id completed above+ // keeps a later replayed event from writing one either.+ if (toolCall.error !== null) {+ return;+ }+files.push({path: `tool_calls/${formatSandboxToolCallTimestamp(draft?.createdAt ?? toolCall.createdAt)}_${draft?.toolName ?? toolCall.toolName}_${toolCall.toolCallId}.json`,content: JSON.stringify(@@ -375,18 +384,28 @@ export function createSandboxToolCallFileAccumulator(return;}+ const error = getString(event, "error") ?? null;+ // Unwrap once and classify the result we would archive, rather than parsing+ // the raw content here and again on the way into the file.+ const response = parseSandboxToolCallValue(+ getString(event, "content"),+ getSandboxInAppAgentMcpToolResultContent,+ );+drafts.delete(toolCallId);completedToolCallIds.add(toolCallId);++ if (getToolFailureMessage(error, response)) {+ return;+ }+files.push({path: `tool_calls/${formatSandboxToolCallTimestamp(draft.createdAt)}_${draft.toolName}_${toolCallId}.json`,content: JSON.stringify({request: parseSandboxToolCallValue(draft.request),- response: parseSandboxToolCallValue(- getString(event, "content"),- getSandboxInAppAgentMcpToolResultContent,- ),- error: getString(event, "error") ?? null,+ response,+ error,},null,2,diff --git a/packages/shared/src/in-app-agent/server/toolErrors.ts b/packages/shared/src/in-app-agent/server/toolErrors.tsindex fd965aa616..2334113a6b 100644--- a/packages/shared/src/in-app-agent/server/toolErrors.ts+++ b/packages/shared/src/in-app-agent/server/toolErrors.ts@@ -25,16 +25,32 @@ export function getToolFailureMessage(return explicitError;}- const normalizedOutput = normalizeToolOutput(output);- if (!isRecord(normalizedOutput)) {+ // The MCP `isError` marker lives on the envelope, which normalizeToolOutput+ // discards when it unwraps a single text item. Check both, or a conventional+ // failure whose text is not JSON reads as a success.+ const parsedOutput =+ typeof output === "string" ? parseJsonOrString(output) : output;+ const normalizedOutput = normalizeToolOutput(parsedOutput);++ if (!hasFailureMarker(normalizedOutput) && !hasFailureMarker(parsedOutput)) {return undefined;}- if (normalizedOutput.error === true || normalizedOutput.isError === true) {- return getStringValue(normalizedOutput.message) ?? "Tool returned an error";- }+ // Prefer the unwrapped text, then either envelope's own message.+ return (+ getStringValue(normalizedOutput) ??+ getMessageValue(normalizedOutput) ??+ getMessageValue(parsedOutput) ??+ "Tool returned an error"+ );+}++function hasFailureMarker(value: unknown): boolean {+ return isRecord(value) && (value.error === true || value.isError === true);+}- return undefined;+function getMessageValue(value: unknown): string | undefined {+ return isRecord(value) ? getStringValue(value.message) : undefined;}export function normalizeToolOutput(output: unknown): unknown {diff --git a/packages/shared/src/in-app-agent/server/tools.ts b/packages/shared/src/in-app-agent/server/tools.tsindex 1aa8c5aa1e..86b641d6da 100644--- a/packages/shared/src/in-app-agent/server/tools.ts+++ b/packages/shared/src/in-app-agent/server/tools.ts@@ -36,6 +36,7 @@ import {IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE,IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,} from "../constants";+import { getToolFailureMessage } from "./toolErrors";type InAppAgentMcpToolApproval = "auto" | "approval";@@ -728,21 +729,31 @@ export function withOptionalSilentMcpOutput(params: {});}+ // Rethrow instead of returning a structured failure: the tool-error+ // chunk rewrite already feeds the error back as a tool result the+ // loop continues from, executeApprovedToolCall classifies+ // approved-tool failures from the throw, and an aborted run must+ // unwind rather than look like one more failed tool call.throw error;}+ const failureMessage = getToolFailureMessage(undefined, result);+if (toolCallId) {params.onToolCallCompleted?.({toolCallId,toolName,request: input,response: result,- error: null,+ error: failureMessage ?? null,createdAt,});}- if (!silent) {+ // Never silence a failure. A tool that reports its error in the result+ // (the MCP `isError` convention) would otherwise be collapsed to a+ // pointer at a tool_calls file that is deliberately not written.+ if (failureMessage || !silent) {return result;}diff --git a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.tsindex 4f67154543..9d768af2e7 100644--- a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts+++ b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts@@ -478,6 +478,24 @@ describe("InAppAgentInstrumentation", () => {},expectedMessage: "Tool returned an error",},+ {+ // The `isError` marker sits on the envelope and the text is not JSON, so+ // the unwrapped content carries no marker of its own.+ name: "isError envelopes wrapping plain text",+ content: JSON.stringify({+ isError: true,+ content: [+ {+ type: "text",+ text: "Error fetching docs page markdown: Failed to fetch https://langfuse.com/not-a-real-page.md: 404",+ },+ ],+ }),+ expectedOutput:+ "Error fetching docs page markdown: Failed to fetch https://langfuse.com/not-a-real-page.md: 404",+ expectedMessage:+ "Error fetching docs page markdown: Failed to fetch https://langfuse.com/not-a-real-page.md: 404",+ },{name: "JSON-encoded approval rejection errors",content: "Tool call was not approved by the user.",diff --git a/web/src/__tests__/server/unit/in-app-agent-sandbox.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-sandbox.servertest.tsindex ec9aaf5919..94f69d38c4 100644--- a/web/src/__tests__/server/unit/in-app-agent-sandbox.servertest.ts+++ b/web/src/__tests__/server/unit/in-app-agent-sandbox.servertest.ts@@ -13,6 +13,23 @@ import { createInAppAgentSandbox } from "@langfuse/shared/in-app-agent/server/saimport { withOptionalSilentMcpOutput } from "@langfuse/shared/in-app-agent/server/tools";import { listObservationsTool } from "@/src/features/mcp/features/observations/tools/listObservations";+// Only presence matters for the tests below: it is what makes+// withOptionalSilentMcpOutput advertise the `silent` parameter.+const dummySandbox = {+ async read() {+ return null;+ },+ async write() {+ return null;+ },+ async edit() {+ return null;+ },+ async bash() {+ return null;+ },+};+describe("in-app agent sandbox", () => {it("redacts silent MCP output from persisted conversation display", async () => {const silentResult = JSON.stringify({@@ -318,20 +335,7 @@ describe("in-app agent sandbox", () => {execute,}),},- sandbox: {- async read() {- return null;- },- async write() {- return null;- },- async edit() {- return null;- },- async bash() {- return null;- },- },+ sandbox: dummySandbox,});const tool = tools.search;@@ -374,20 +378,7 @@ describe("in-app agent sandbox", () => {}),},},- sandbox: {- async read() {- return null;- },- async write() {- return null;- },- async edit() {- return null;- },- async bash() {- return null;- },- },+ sandbox: dummySandbox,});if (!tools.listObservations.inputSchema) {@@ -454,4 +445,111 @@ describe("in-app agent sandbox", () => {expect("silent" in tools.search.inputSchema.shape).toBe(false);});++ it("does not silence an MCP result that reports its own error", async () => {+ // Verbatim shape of a real failure from the langfuse docs MCP server: the+ // `isError` marker sits on the envelope and the text is not JSON, so+ // classifying the unwrapped content alone reads it as a success.+ const mcpErrorResult = {+ isError: true,+ content: [+ {+ type: "text",+ text: "Error fetching docs page markdown: Failed to fetch https://langfuse.com/not-a-real-page.md: 404",+ },+ ],+ };+ const toolCallFiles = createSandboxToolCallFileAccumulator([]);+ const tools = withOptionalSilentMcpOutput({+ tools: {+ search: new Tool({+ id: "search",+ description: "Search",+ inputSchema: z.object({ query: z.string() }),+ execute: async () => mcpErrorResult,+ }),+ },+ sandbox: dummySandbox,+ onToolCallCompleted: toolCallFiles.processToolCall,+ });++ const output = await tools.search.execute?.(+ { query: "test", silent: true },+ { agent: { toolCallId: "tool-call-1" } } as never,+ );++ // Not wrapped, so the model reads the failure inline rather than being+ // pointed at a tool_calls file that is never written for failures.+ expect(output).toEqual(mcpErrorResult);+ expect(tools.search.toModelOutput?.(output)).toEqual(mcpErrorResult);+ expect(toolCallFiles.getFiles()).toEqual([]);+ });++ it("rethrows a thrown silent MCP error and writes no sandbox file", async () => {+ const toolCallFiles = createSandboxToolCallFileAccumulator([]);+ const tools = withOptionalSilentMcpOutput({+ tools: {+ search: new Tool({+ id: "search",+ description: "Search",+ inputSchema: z.object({ query: z.string() }),+ execute: async () => {+ throw new Error(+ "McpError -32602: Validation failed: Required at view",+ );+ },+ }),+ },+ sandbox: dummySandbox,+ onToolCallCompleted: toolCallFiles.processToolCall,+ });++ // The throw must survive the wrapper: the adapter's tool-error rewrite+ // feeds it back to the model, and approved-tool failures are classified+ // from it.+ await expect(+ tools.search.execute?.({ query: "test", silent: true }, {+ agent: { toolCallId: "tool-call-2" },+ } as never),+ ).rejects.toThrow("McpError -32602: Validation failed: Required at view");+ expect(toolCallFiles.getFiles()).toEqual([]);+ });++ it("does not export failed tool calls into sandbox tool_calls files", () => {+ const files = getSandboxToolCallFiles([+ {+ createdAt: new Date("2026-07-02T12:00:00.000Z"),+ runId: "run-1",+ event: {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-call-1",+ toolCallName: "langfuse_queryMetrics",+ },+ },+ {+ createdAt: new Date("2026-07-02T12:00:00.100Z"),+ runId: "run-1",+ event: {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-call-1",+ delta: '{"silent":true}',+ },+ },+ {+ createdAt: new Date("2026-07-02T12:00:00.200Z"),+ runId: "run-1",+ event: {+ type: EventType.TOOL_CALL_RESULT,+ toolCallId: "tool-call-1",+ content: JSON.stringify({+ error: true,+ message:+ "Tool input validation failed for langfuse_queryMetrics. Please fix the following errors and try again:\n- root: must have required property 'view'",+ }),+ },+ },+ ]);++ expect(files).toEqual([]);+ });});