Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): stop silencing MCP tool failures into the sandbox (#16082)

Lysander Kiesel62033434be81
diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.ts
index 87c16dfcab..bf1ef0a3bd 100644
--- a/packages/shared/src/in-app-agent/server/persistence.ts
+++ b/packages/shared/src/in-app-agent/server/persistence.ts
@@ -43,6 +43,7 @@ import {
getSandboxInAppAgentMcpToolResultContent,
IN_APP_AGENT_SANDBOX_TOOL_NAMES,
} from "./tools";
+import { getToolFailureMessage } from "./toolErrors";
export const ACTIVE_RUN_CONFLICT_MESSAGE =
"Assistant is already responding in this conversation";
@@ -315,6 +316,14 @@ export function createSandboxToolCallFileAccumulator(
const draft = drafts.get(toolCall.toolCallId);
drafts.delete(toolCall.toolCallId);
completedToolCallIds.add(toolCall.toolCallId);
+
+ // The MCP wrapper already classified the failure onto `error`, so failures
+ // never become sandbox tool_calls files. Marking the id completed above
+ // keeps a later replayed event from writing one either.
+ if (toolCall.error !== null) {
+ return;
+ }
+
files.push({
path: `tool_calls/${formatSandboxToolCallTimestamp(draft?.createdAt ?? toolCall.createdAt)}_${draft?.toolName ?? toolCall.toolName}_${toolCall.toolCallId}.json`,
content: JSON.stringify(
@@ -375,18 +384,28 @@ export function createSandboxToolCallFileAccumulator(
return;
}
+ const error = getString(event, "error") ?? null;
+ // Unwrap once and classify the result we would archive, rather than parsing
+ // the raw content here and again on the way into the file.
+ const response = parseSandboxToolCallValue(
+ getString(event, "content"),
+ getSandboxInAppAgentMcpToolResultContent,
+ );
+
drafts.delete(toolCallId);
completedToolCallIds.add(toolCallId);
+
+ if (getToolFailureMessage(error, response)) {
+ return;
+ }
+
files.push({
path: `tool_calls/${formatSandboxToolCallTimestamp(draft.createdAt)}_${draft.toolName}_${toolCallId}.json`,
content: JSON.stringify(
{
request: parseSandboxToolCallValue(draft.request),
- response: parseSandboxToolCallValue(
- getString(event, "content"),
- getSandboxInAppAgentMcpToolResultContent,
- ),
- error: getString(event, "error") ?? null,
+ response,
+ error,
},
null,
2,
diff --git a/packages/shared/src/in-app-agent/server/toolErrors.ts b/packages/shared/src/in-app-agent/server/toolErrors.ts
index fd965aa616..2334113a6b 100644
--- a/packages/shared/src/in-app-agent/server/toolErrors.ts
+++ b/packages/shared/src/in-app-agent/server/toolErrors.ts
@@ -25,16 +25,32 @@ export function getToolFailureMessage(
return explicitError;
}
- const normalizedOutput = normalizeToolOutput(output);
- if (!isRecord(normalizedOutput)) {
+ // The MCP `isError` marker lives on the envelope, which normalizeToolOutput
+ // discards when it unwraps a single text item. Check both, or a conventional
+ // failure whose text is not JSON reads as a success.
+ const parsedOutput =
+ typeof output === "string" ? parseJsonOrString(output) : output;
+ const normalizedOutput = normalizeToolOutput(parsedOutput);
+
+ if (!hasFailureMarker(normalizedOutput) && !hasFailureMarker(parsedOutput)) {
return undefined;
}
- if (normalizedOutput.error === true || normalizedOutput.isError === true) {
- return getStringValue(normalizedOutput.message) ?? "Tool returned an error";
- }
+ // Prefer the unwrapped text, then either envelope's own message.
+ return (
+ getStringValue(normalizedOutput) ??
+ getMessageValue(normalizedOutput) ??
+ getMessageValue(parsedOutput) ??
+ "Tool returned an error"
+ );
+}
+
+function hasFailureMarker(value: unknown): boolean {
+ return isRecord(value) && (value.error === true || value.isError === true);
+}
- return undefined;
+function getMessageValue(value: unknown): string | undefined {
+ return isRecord(value) ? getStringValue(value.message) : undefined;
}
export function normalizeToolOutput(output: unknown): unknown {
diff --git a/packages/shared/src/in-app-agent/server/tools.ts b/packages/shared/src/in-app-agent/server/tools.ts
index 1aa8c5aa1e..86b641d6da 100644
--- a/packages/shared/src/in-app-agent/server/tools.ts
+++ b/packages/shared/src/in-app-agent/server/tools.ts
@@ -36,6 +36,7 @@ import {
IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE,
IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,
} from "../constants";
+import { getToolFailureMessage } from "./toolErrors";
type InAppAgentMcpToolApproval = "auto" | "approval";
@@ -728,21 +729,31 @@ export function withOptionalSilentMcpOutput(params: {
});
}
+ // Rethrow instead of returning a structured failure: the tool-error
+ // chunk rewrite already feeds the error back as a tool result the
+ // loop continues from, executeApprovedToolCall classifies
+ // approved-tool failures from the throw, and an aborted run must
+ // unwind rather than look like one more failed tool call.
throw error;
}
+ const failureMessage = getToolFailureMessage(undefined, result);
+
if (toolCallId) {
params.onToolCallCompleted?.({
toolCallId,
toolName,
request: input,
response: result,
- error: null,
+ error: failureMessage ?? null,
createdAt,
});
}
- if (!silent) {
+ // Never silence a failure. A tool that reports its error in the result
+ // (the MCP `isError` convention) would otherwise be collapsed to a
+ // pointer at a tool_calls file that is deliberately not written.
+ if (failureMessage || !silent) {
return result;
}
diff --git a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
index 4f67154543..9d768af2e7 100644
--- a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
+++ b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
@@ -478,6 +478,24 @@ describe("InAppAgentInstrumentation", () => {
},
expectedMessage: "Tool returned an error",
},
+ {
+ // The `isError` marker sits on the envelope and the text is not JSON, so
+ // the unwrapped content carries no marker of its own.
+ name: "isError envelopes wrapping plain text",
+ content: JSON.stringify({
+ isError: true,
+ content: [
+ {
+ type: "text",
+ text: "Error fetching docs page markdown: Failed to fetch https://langfuse.com/not-a-real-page.md: 404",
+ },
+ ],
+ }),
+ expectedOutput:
+ "Error fetching docs page markdown: Failed to fetch https://langfuse.com/not-a-real-page.md: 404",
+ expectedMessage:
+ "Error fetching docs page markdown: Failed to fetch https://langfuse.com/not-a-real-page.md: 404",
+ },
{
name: "JSON-encoded approval rejection errors",
content: "Tool call was not approved by the user.",
diff --git a/web/src/__tests__/server/unit/in-app-agent-sandbox.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-sandbox.servertest.ts
index ec9aaf5919..94f69d38c4 100644
--- a/web/src/__tests__/server/unit/in-app-agent-sandbox.servertest.ts
+++ b/web/src/__tests__/server/unit/in-app-agent-sandbox.servertest.ts
@@ -13,6 +13,23 @@ import { createInAppAgentSandbox } from "@langfuse/shared/in-app-agent/server/sa
import { withOptionalSilentMcpOutput } from "@langfuse/shared/in-app-agent/server/tools";
import { listObservationsTool } from "@/src/features/mcp/features/observations/tools/listObservations";
+// Only presence matters for the tests below: it is what makes
+// withOptionalSilentMcpOutput advertise the `silent` parameter.
+const dummySandbox = {
+ async read() {
+ return null;
+ },
+ async write() {
+ return null;
+ },
+ async edit() {
+ return null;
+ },
+ async bash() {
+ return null;
+ },
+};
+
describe("in-app agent sandbox", () => {
it("redacts silent MCP output from persisted conversation display", async () => {
const silentResult = JSON.stringify({
@@ -318,20 +335,7 @@ describe("in-app agent sandbox", () => {
execute,
}),
},
- sandbox: {
- async read() {
- return null;
- },
- async write() {
- return null;
- },
- async edit() {
- return null;
- },
- async bash() {
- return null;
- },
- },
+ sandbox: dummySandbox,
});
const tool = tools.search;
@@ -374,20 +378,7 @@ describe("in-app agent sandbox", () => {
}),
},
},
- sandbox: {
- async read() {
- return null;
- },
- async write() {
- return null;
- },
- async edit() {
- return null;
- },
- async bash() {
- return null;
- },
- },
+ sandbox: dummySandbox,
});
if (!tools.listObservations.inputSchema) {
@@ -454,4 +445,111 @@ describe("in-app agent sandbox", () => {
expect("silent" in tools.search.inputSchema.shape).toBe(false);
});
+
+ it("does not silence an MCP result that reports its own error", async () => {
+ // Verbatim shape of a real failure from the langfuse docs MCP server: the
+ // `isError` marker sits on the envelope and the text is not JSON, so
+ // classifying the unwrapped content alone reads it as a success.
+ const mcpErrorResult = {
+ isError: true,
+ content: [
+ {
+ type: "text",
+ text: "Error fetching docs page markdown: Failed to fetch https://langfuse.com/not-a-real-page.md: 404",
+ },
+ ],
+ };
+ const toolCallFiles = createSandboxToolCallFileAccumulator([]);
+ const tools = withOptionalSilentMcpOutput({
+ tools: {
+ search: new Tool({
+ id: "search",
+ description: "Search",
+ inputSchema: z.object({ query: z.string() }),
+ execute: async () => mcpErrorResult,
+ }),
+ },
+ sandbox: dummySandbox,
+ onToolCallCompleted: toolCallFiles.processToolCall,
+ });
+
+ const output = await tools.search.execute?.(
+ { query: "test", silent: true },
+ { agent: { toolCallId: "tool-call-1" } } as never,
+ );
+
+ // Not wrapped, so the model reads the failure inline rather than being
+ // pointed at a tool_calls file that is never written for failures.
+ expect(output).toEqual(mcpErrorResult);
+ expect(tools.search.toModelOutput?.(output)).toEqual(mcpErrorResult);
+ expect(toolCallFiles.getFiles()).toEqual([]);
+ });
+
+ it("rethrows a thrown silent MCP error and writes no sandbox file", async () => {
+ const toolCallFiles = createSandboxToolCallFileAccumulator([]);
+ const tools = withOptionalSilentMcpOutput({
+ tools: {
+ search: new Tool({
+ id: "search",
+ description: "Search",
+ inputSchema: z.object({ query: z.string() }),
+ execute: async () => {
+ throw new Error(
+ "McpError -32602: Validation failed: Required at view",
+ );
+ },
+ }),
+ },
+ sandbox: dummySandbox,
+ onToolCallCompleted: toolCallFiles.processToolCall,
+ });
+
+ // The throw must survive the wrapper: the adapter's tool-error rewrite
+ // feeds it back to the model, and approved-tool failures are classified
+ // from it.
+ await expect(
+ tools.search.execute?.({ query: "test", silent: true }, {
+ agent: { toolCallId: "tool-call-2" },
+ } as never),
+ ).rejects.toThrow("McpError -32602: Validation failed: Required at view");
+ expect(toolCallFiles.getFiles()).toEqual([]);
+ });
+
+ it("does not export failed tool calls into sandbox tool_calls files", () => {
+ const files = getSandboxToolCallFiles([
+ {
+ createdAt: new Date("2026-07-02T12:00:00.000Z"),
+ runId: "run-1",
+ event: {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-call-1",
+ toolCallName: "langfuse_queryMetrics",
+ },
+ },
+ {
+ createdAt: new Date("2026-07-02T12:00:00.100Z"),
+ runId: "run-1",
+ event: {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-call-1",
+ delta: '{"silent":true}',
+ },
+ },
+ {
+ createdAt: new Date("2026-07-02T12:00:00.200Z"),
+ runId: "run-1",
+ event: {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-call-1",
+ content: JSON.stringify({
+ error: true,
+ message:
+ "Tool input validation failed for langfuse_queryMetrics. Please fix the following errors and try again:\n- root: must have required property 'view'",
+ }),
+ },
+ },
+ ]);
+
+ expect(files).toEqual([]);
+ });
});