Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): map MCP tool results to AI SDK model output (#16618)

Lysander Kiesel9f84a4537459
diff --git a/packages/shared/src/in-app-agent/server/toolResults.ts b/packages/shared/src/in-app-agent/server/toolResults.ts
index c781e019a1..6768acc20a 100644
--- a/packages/shared/src/in-app-agent/server/toolResults.ts
+++ b/packages/shared/src/in-app-agent/server/toolResults.ts
@@ -5,6 +5,7 @@ import {
IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,
} from "../constants";
import type { AgUiEvent } from "../schema";
+import { isRecord } from "./toolErrors";
export type SilentInAppAgentMcpToolOutput = {
type: typeof IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE;
@@ -71,6 +72,63 @@ export function isSilentInAppAgentMcpToolOutput(
);
}
+export type AiSdkToolModelOutput = {
+ type: string;
+ value: unknown;
+};
+
+/**
+ * Map tool execute results to the LanguageModelV3 `{ type, value }` shape.
+ * Providers serialize `output.value`; a raw MCP `{ content }` envelope has
+ * no `.value` and becomes an empty tool result on the wire.
+ */
+export function toAiSdkToolModelOutput(output: unknown): AiSdkToolModelOutput {
+ if (typeof output === "string") {
+ return { type: "text", value: output };
+ }
+
+ if (isSilentInAppAgentMcpToolOutput(output)) {
+ if (output.toolCallId && output.toolName) {
+ return {
+ type: "text",
+ value: getInAppAgentSilentMcpOutputMessage(
+ output.toolName,
+ output.toolCallId,
+ ),
+ };
+ }
+
+ return toAiSdkToolModelOutput(output.output);
+ }
+
+ if (
+ isRecord(output) &&
+ typeof output.type === "string" &&
+ "value" in output
+ ) {
+ return { type: output.type, value: output.value };
+ }
+
+ return { type: "json", value: output };
+}
+
+/** AG-UI tool messages store a string. Unwrap LanguageModelV3 `{ type, value }`. */
+export function toAgUiToolResultContent(value: unknown): string {
+ if (typeof value === "string") {
+ return value;
+ }
+
+ if (isRecord(value) && typeof value.type === "string" && "value" in value) {
+ return toAgUiToolResultContent(value.value);
+ }
+
+ try {
+ return JSON.stringify(value ?? null);
+ } catch {
+ return String(value);
+ }
+}
+
/** Withhold private persisted event payloads from browser-facing streams. */
export function toPublicInAppAgentEvent(event: AgUiEvent): AgUiEvent {
if (event.type === EventType.RUN_STARTED && event.input !== undefined) {
diff --git a/worker/src/features/in-app-agent/runtime/human-in-the-loop.test.ts b/worker/src/features/in-app-agent/runtime/human-in-the-loop.test.ts
new file mode 100644
index 0000000000..a28cd21f0b
--- /dev/null
+++ b/worker/src/features/in-app-agent/runtime/human-in-the-loop.test.ts
@@ -0,0 +1,93 @@
+import { EventType } from "@ag-ui/core";
+import { describe, expect, it } from "vitest";
+
+import { createManualToolApprovalRunInput } from "./human-in-the-loop";
+import type { AgUiRunAgentInput } from "./types";
+
+const approvalRequest = {
+ type: "tool_approval_request" as const,
+ toolCallId: "tool-call-1",
+ toolName: "langfuse_createScoreConfig",
+ args: { name: "readiness", dataType: "NUMERIC" },
+ runId: "interrupted-run-1",
+};
+
+function resumeInput(): AgUiRunAgentInput {
+ return {
+ threadId: "conversation-1",
+ runId: "run-2",
+ messages: [
+ {
+ id: "user-message-1",
+ role: "user",
+ content: "create a readiness score config",
+ },
+ ],
+ tools: [],
+ context: [],
+ forwardedProps: {
+ command: {
+ resume: {
+ approved: true,
+ approvalRequest,
+ },
+ },
+ },
+ };
+}
+
+function toolResultContent(events: { type: string; content?: string }[]) {
+ return events.find((event) => event.type === EventType.TOOL_CALL_RESULT)
+ ?.content;
+}
+
+describe("createManualToolApprovalRunInput", () => {
+ it("resumes a non-silent approval with the inner payload, not the LanguageModelV3 wrapper", async () => {
+ const result = {
+ id: "score-config-1",
+ name: "readiness",
+ dataType: "NUMERIC",
+ };
+
+ const { input, syntheticEvents } = await createManualToolApprovalRunInput({
+ input: resumeInput(),
+ executeToolCall: async () => ({
+ result,
+ modelResult: { type: "json", value: result },
+ }),
+ });
+ const toolMessage = input.messages.find(
+ (message) => message.role === "tool",
+ );
+
+ expect(toolMessage?.content).toEqual(JSON.stringify(result));
+ expect(toolResultContent(syntheticEvents)).toEqual(JSON.stringify(result));
+ });
+
+ it("persists a silent approval envelope and resumes with the sandbox path", async () => {
+ const envelope = {
+ type: "silent-mcp-output",
+ toolCallId: "tool-call-1",
+ toolName: "langfuse_createScoreConfig",
+ output: { id: "score-config-1", secret: "full-tool-output" },
+ };
+ const path =
+ "Output saved to /workspace/tool_calls/langfuse_createScoreConfig_tool-call-1.json";
+
+ const { input, syntheticEvents } = await createManualToolApprovalRunInput({
+ input: resumeInput(),
+ executeToolCall: async () => ({
+ result: envelope,
+ modelResult: { type: "text", value: path },
+ }),
+ });
+ const toolMessage = input.messages.find(
+ (message) => message.role === "tool",
+ );
+
+ expect(toolMessage?.content).toEqual(path);
+ expect(toolResultContent(syntheticEvents)).toEqual(
+ JSON.stringify(envelope),
+ );
+ });
+});
diff --git a/worker/src/features/in-app-agent/runtime/human-in-the-loop.ts b/worker/src/features/in-app-agent/runtime/human-in-the-loop.ts
index f332a88f2f..1a4a5f4da5 100644
--- a/worker/src/features/in-app-agent/runtime/human-in-the-loop.ts
+++ b/worker/src/features/in-app-agent/runtime/human-in-the-loop.ts
@@ -5,6 +5,7 @@ import {
type AgUiMessage,
type InAppAgentToolApprovalRequest,
} from "@langfuse/shared/in-app-agent";
+import { toAgUiToolResultContent } from "@langfuse/shared/in-app-agent/server/toolResults";
import type { AgUiRunAgentInput, ResumeForwardedProps } from "./types";
const MANUAL_TOOL_APPROVAL_REJECTION_MESSAGE =
@@ -85,7 +86,7 @@ export async function createManualToolApprovalRunInput(params: {
const toolResultContent = serializeToolResultContent(toolResult);
const assistantMessage =
createManualToolCallAssistantMessage(approvalRequest);
- const modelToolResultContent = serializeToolResultContent(modelToolResult);
+ const modelToolResultContent = toAgUiToolResultContent(modelToolResult);
// A successful tool call needs no guidance: the assistant tool call plus its
// tool result already carry the outcome. Only the error path needs to tell
// the model how to proceed.
diff --git a/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts b/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts
index d6bab93def..83c2abbe4e 100644
--- a/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts
+++ b/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts
@@ -181,9 +181,10 @@ describe("in-app agent sandbox", () => {
toolCallId: "tool-call-1",
toolName: "search",
});
- expect(tool.toModelOutput?.(output)).toBe(
- "Output saved to /workspace/tool_calls/search_tool-call-1.json",
- );
+ expect(tool.toModelOutput?.(output)).toEqual({
+ type: "text",
+ value: "Output saved to /workspace/tool_calls/search_tool-call-1.json",
+ });
});
it("supports silent output for the listObservations JSON Schema", async () => {
@@ -311,7 +312,10 @@ describe("in-app agent sandbox", () => {
// Not wrapped, so the model reads the failure inline rather than being
// pointed at a tool_calls file that is never written for failures.
expect(output).toEqual(mcpErrorResult);
- expect(tools.search.toModelOutput?.(output)).toEqual(mcpErrorResult);
+ expect(tools.search.toModelOutput?.(output)).toEqual({
+ type: "json",
+ value: mcpErrorResult,
+ });
expect(toolCallFiles.getFiles()).toEqual([]);
});
diff --git a/worker/src/features/in-app-agent/runtime/tools.modelOutput.test.ts b/worker/src/features/in-app-agent/runtime/tools.modelOutput.test.ts
new file mode 100644
index 0000000000..7e7d446a4d
--- /dev/null
+++ b/worker/src/features/in-app-agent/runtime/tools.modelOutput.test.ts
@@ -0,0 +1,67 @@
+import { Tool } from "@mastra/core/tools";
+import { describe, expect, it } from "vitest";
+import { z } from "zod";
+
+import { withOptionalSilentMcpOutput } from "./tools";
+
+// Shape of a real MCP CallToolResult from getObservationFilterValues: the
+// VALUES JSON lives in content[].text, with no AI SDK `{ type, value }`.
+const FILTER_VALUES_JSON = JSON.stringify({
+ type: "VALUES",
+ column: "name",
+ values: [
+ { value: "Codex Turn", count: 12 },
+ { value: "generation", count: 40 },
+ ],
+ meta: {},
+});
+
+const FILTER_VALUES_MCP_RESULT = {
+ content: [{ type: "text", text: FILTER_VALUES_JSON }],
+};
+
+const dummySandbox = {
+ async read() {
+ return null;
+ },
+ async write() {
+ return null;
+ },
+ async edit() {
+ return null;
+ },
+ async bash() {
+ return null;
+ },
+};
+
+describe("in-app agent MCP toModelOutput", () => {
+ it("maps an MCP CallToolResult to AI SDK { type, value }", async () => {
+ const tools = withOptionalSilentMcpOutput({
+ tools: {
+ langfuse_getObservationFilterValues: new Tool({
+ id: "langfuse_getObservationFilterValues",
+ description: "Get observation filter values",
+ inputSchema: z.object({ column: z.string() }),
+ execute: async () => FILTER_VALUES_MCP_RESULT,
+ }),
+ },
+ sandbox: dummySandbox,
+ });
+ const tool = tools.langfuse_getObservationFilterValues;
+ const result = await tool.execute?.({ column: "name" }, {
+ agent: { toolCallId: "tooluse_filter_values" },
+ } as never);
+
+ const output = tool.toModelOutput?.(result);
+
+ // execute() stays the MCP envelope for events + sandbox rewind.
+ // toModelOutput only wraps it so providers can serialize output.value.
+ expect(result).toEqual(FILTER_VALUES_MCP_RESULT);
+ expect(output).toEqual({
+ type: "json",
+ value: FILTER_VALUES_MCP_RESULT,
+ });
+ expect(JSON.stringify(output)).toContain("Codex Turn");
+ });
+});
diff --git a/worker/src/features/in-app-agent/runtime/tools.ts b/worker/src/features/in-app-agent/runtime/tools.ts
index 9a293467dd..bf821b9598 100644
--- a/worker/src/features/in-app-agent/runtime/tools.ts
+++ b/worker/src/features/in-app-agent/runtime/tools.ts
@@ -39,8 +39,8 @@ import {
IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,
} from "@langfuse/shared/in-app-agent";
import {
- getInAppAgentSilentMcpOutputMessage,
isSilentInAppAgentMcpToolOutput,
+ toAiSdkToolModelOutput,
type CompletedInAppAgentMcpToolCall,
type SilentInAppAgentMcpToolOutput,
} from "@langfuse/shared/in-app-agent/server/toolResults";
@@ -219,19 +219,12 @@ export function withOptionalSilentMcpOutput(params: {
};
tool.toModelOutput = (output) => {
if (isSilentInAppAgentMcpToolOutput(output)) {
- if (!output.toolCallId) {
- return output.output;
- }
-
- return output.toolName
- ? getInAppAgentSilentMcpOutputMessage(
- output.toolName,
- output.toolCallId,
- )
- : output.output;
+ return toAiSdkToolModelOutput(output);
}
- return toModelOutput ? toModelOutput(output) : output;
+ return toAiSdkToolModelOutput(
+ toModelOutput ? toModelOutput(output) : output,
+ );
};
return [toolName, tool];