🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): map MCP tool results to AI SDK model output (#16618)
Lysander Kiesel9f84a4537459
diff --git a/packages/shared/src/in-app-agent/server/toolResults.ts b/packages/shared/src/in-app-agent/server/toolResults.tsindex c781e019a1..6768acc20a 100644--- a/packages/shared/src/in-app-agent/server/toolResults.ts+++ b/packages/shared/src/in-app-agent/server/toolResults.ts@@ -5,6 +5,7 @@ import {IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,} from "../constants";import type { AgUiEvent } from "../schema";+import { isRecord } from "./toolErrors";export type SilentInAppAgentMcpToolOutput = {type: typeof IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE;@@ -71,6 +72,63 @@ export function isSilentInAppAgentMcpToolOutput();}+export type AiSdkToolModelOutput = {+ type: string;+ value: unknown;+};++/**+ * Map tool execute results to the LanguageModelV3 `{ type, value }` shape.+ * Providers serialize `output.value`; a raw MCP `{ content }` envelope has+ * no `.value` and becomes an empty tool result on the wire.+ */+export function toAiSdkToolModelOutput(output: unknown): AiSdkToolModelOutput {+ if (typeof output === "string") {+ return { type: "text", value: output };+ }++ if (isSilentInAppAgentMcpToolOutput(output)) {+ if (output.toolCallId && output.toolName) {+ return {+ type: "text",+ value: getInAppAgentSilentMcpOutputMessage(+ output.toolName,+ output.toolCallId,+ ),+ };+ }++ return toAiSdkToolModelOutput(output.output);+ }++ if (+ isRecord(output) &&+ typeof output.type === "string" &&+ "value" in output+ ) {+ return { type: output.type, value: output.value };+ }++ return { type: "json", value: output };+}++/** AG-UI tool messages store a string. Unwrap LanguageModelV3 `{ type, value }`. */+export function toAgUiToolResultContent(value: unknown): string {+ if (typeof value === "string") {+ return value;+ }++ if (isRecord(value) && typeof value.type === "string" && "value" in value) {+ return toAgUiToolResultContent(value.value);+ }++ try {+ return JSON.stringify(value ?? null);+ } catch {+ return String(value);+ }+}+/** Withhold private persisted event payloads from browser-facing streams. */export function toPublicInAppAgentEvent(event: AgUiEvent): AgUiEvent {if (event.type === EventType.RUN_STARTED && event.input !== undefined) {diff --git a/worker/src/features/in-app-agent/runtime/human-in-the-loop.test.ts b/worker/src/features/in-app-agent/runtime/human-in-the-loop.test.tsnew file mode 100644index 0000000000..a28cd21f0b--- /dev/null+++ b/worker/src/features/in-app-agent/runtime/human-in-the-loop.test.ts@@ -0,0 +1,93 @@+import { EventType } from "@ag-ui/core";+import { describe, expect, it } from "vitest";++import { createManualToolApprovalRunInput } from "./human-in-the-loop";+import type { AgUiRunAgentInput } from "./types";++const approvalRequest = {+ type: "tool_approval_request" as const,+ toolCallId: "tool-call-1",+ toolName: "langfuse_createScoreConfig",+ args: { name: "readiness", dataType: "NUMERIC" },+ runId: "interrupted-run-1",+};++function resumeInput(): AgUiRunAgentInput {+ return {+ threadId: "conversation-1",+ runId: "run-2",+ messages: [+ {+ id: "user-message-1",+ role: "user",+ content: "create a readiness score config",+ },+ ],+ tools: [],+ context: [],+ forwardedProps: {+ command: {+ resume: {+ approved: true,+ approvalRequest,+ },+ },+ },+ };+}++function toolResultContent(events: { type: string; content?: string }[]) {+ return events.find((event) => event.type === EventType.TOOL_CALL_RESULT)+ ?.content;+}++describe("createManualToolApprovalRunInput", () => {+ it("resumes a non-silent approval with the inner payload, not the LanguageModelV3 wrapper", async () => {+ const result = {+ id: "score-config-1",+ name: "readiness",+ dataType: "NUMERIC",+ };++ const { input, syntheticEvents } = await createManualToolApprovalRunInput({+ input: resumeInput(),+ executeToolCall: async () => ({+ result,+ modelResult: { type: "json", value: result },+ }),+ });+ const toolMessage = input.messages.find(+ (message) => message.role === "tool",+ );++ expect(toolMessage?.content).toEqual(JSON.stringify(result));+ expect(toolResultContent(syntheticEvents)).toEqual(JSON.stringify(result));+ });++ it("persists a silent approval envelope and resumes with the sandbox path", async () => {+ const envelope = {+ type: "silent-mcp-output",+ toolCallId: "tool-call-1",+ toolName: "langfuse_createScoreConfig",+ output: { id: "score-config-1", secret: "full-tool-output" },+ };+ const path =+ "Output saved to /workspace/tool_calls/langfuse_createScoreConfig_tool-call-1.json";++ const { input, syntheticEvents } = await createManualToolApprovalRunInput({+ input: resumeInput(),+ executeToolCall: async () => ({+ result: envelope,+ modelResult: { type: "text", value: path },+ }),+ });+ const toolMessage = input.messages.find(+ (message) => message.role === "tool",+ );++ expect(toolMessage?.content).toEqual(path);+ expect(toolResultContent(syntheticEvents)).toEqual(+ JSON.stringify(envelope),+ );+ });+});diff --git a/worker/src/features/in-app-agent/runtime/human-in-the-loop.ts b/worker/src/features/in-app-agent/runtime/human-in-the-loop.tsindex f332a88f2f..1a4a5f4da5 100644--- a/worker/src/features/in-app-agent/runtime/human-in-the-loop.ts+++ b/worker/src/features/in-app-agent/runtime/human-in-the-loop.ts@@ -5,6 +5,7 @@ import {type AgUiMessage,type InAppAgentToolApprovalRequest,} from "@langfuse/shared/in-app-agent";+import { toAgUiToolResultContent } from "@langfuse/shared/in-app-agent/server/toolResults";import type { AgUiRunAgentInput, ResumeForwardedProps } from "./types";const MANUAL_TOOL_APPROVAL_REJECTION_MESSAGE =@@ -85,7 +86,7 @@ export async function createManualToolApprovalRunInput(params: {const toolResultContent = serializeToolResultContent(toolResult);const assistantMessage =createManualToolCallAssistantMessage(approvalRequest);- const modelToolResultContent = serializeToolResultContent(modelToolResult);+ const modelToolResultContent = toAgUiToolResultContent(modelToolResult);// A successful tool call needs no guidance: the assistant tool call plus its// tool result already carry the outcome. Only the error path needs to tell// the model how to proceed.diff --git a/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts b/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.tsindex d6bab93def..83c2abbe4e 100644--- a/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts+++ b/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts@@ -181,9 +181,10 @@ describe("in-app agent sandbox", () => {toolCallId: "tool-call-1",toolName: "search",});- expect(tool.toModelOutput?.(output)).toBe(- "Output saved to /workspace/tool_calls/search_tool-call-1.json",- );+ expect(tool.toModelOutput?.(output)).toEqual({+ type: "text",+ value: "Output saved to /workspace/tool_calls/search_tool-call-1.json",+ });});it("supports silent output for the listObservations JSON Schema", async () => {@@ -311,7 +312,10 @@ describe("in-app agent sandbox", () => {// Not wrapped, so the model reads the failure inline rather than being// pointed at a tool_calls file that is never written for failures.expect(output).toEqual(mcpErrorResult);- expect(tools.search.toModelOutput?.(output)).toEqual(mcpErrorResult);+ expect(tools.search.toModelOutput?.(output)).toEqual({+ type: "json",+ value: mcpErrorResult,+ });expect(toolCallFiles.getFiles()).toEqual([]);});diff --git a/worker/src/features/in-app-agent/runtime/tools.modelOutput.test.ts b/worker/src/features/in-app-agent/runtime/tools.modelOutput.test.tsnew file mode 100644index 0000000000..7e7d446a4d--- /dev/null+++ b/worker/src/features/in-app-agent/runtime/tools.modelOutput.test.ts@@ -0,0 +1,67 @@+import { Tool } from "@mastra/core/tools";+import { describe, expect, it } from "vitest";+import { z } from "zod";++import { withOptionalSilentMcpOutput } from "./tools";++// Shape of a real MCP CallToolResult from getObservationFilterValues: the+// VALUES JSON lives in content[].text, with no AI SDK `{ type, value }`.+const FILTER_VALUES_JSON = JSON.stringify({+ type: "VALUES",+ column: "name",+ values: [+ { value: "Codex Turn", count: 12 },+ { value: "generation", count: 40 },+ ],+ meta: {},+});++const FILTER_VALUES_MCP_RESULT = {+ content: [{ type: "text", text: FILTER_VALUES_JSON }],+};++const dummySandbox = {+ async read() {+ return null;+ },+ async write() {+ return null;+ },+ async edit() {+ return null;+ },+ async bash() {+ return null;+ },+};++describe("in-app agent MCP toModelOutput", () => {+ it("maps an MCP CallToolResult to AI SDK { type, value }", async () => {+ const tools = withOptionalSilentMcpOutput({+ tools: {+ langfuse_getObservationFilterValues: new Tool({+ id: "langfuse_getObservationFilterValues",+ description: "Get observation filter values",+ inputSchema: z.object({ column: z.string() }),+ execute: async () => FILTER_VALUES_MCP_RESULT,+ }),+ },+ sandbox: dummySandbox,+ });+ const tool = tools.langfuse_getObservationFilterValues;+ const result = await tool.execute?.({ column: "name" }, {+ agent: { toolCallId: "tooluse_filter_values" },+ } as never);++ const output = tool.toModelOutput?.(result);++ // execute() stays the MCP envelope for events + sandbox rewind.+ // toModelOutput only wraps it so providers can serialize output.value.+ expect(result).toEqual(FILTER_VALUES_MCP_RESULT);+ expect(output).toEqual({+ type: "json",+ value: FILTER_VALUES_MCP_RESULT,+ });+ expect(JSON.stringify(output)).toContain("Codex Turn");+ });+});diff --git a/worker/src/features/in-app-agent/runtime/tools.ts b/worker/src/features/in-app-agent/runtime/tools.tsindex 9a293467dd..bf821b9598 100644--- a/worker/src/features/in-app-agent/runtime/tools.ts+++ b/worker/src/features/in-app-agent/runtime/tools.ts@@ -39,8 +39,8 @@ import {IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,} from "@langfuse/shared/in-app-agent";import {- getInAppAgentSilentMcpOutputMessage,isSilentInAppAgentMcpToolOutput,+ toAiSdkToolModelOutput,type CompletedInAppAgentMcpToolCall,type SilentInAppAgentMcpToolOutput,} from "@langfuse/shared/in-app-agent/server/toolResults";@@ -219,19 +219,12 @@ export function withOptionalSilentMcpOutput(params: {};tool.toModelOutput = (output) => {if (isSilentInAppAgentMcpToolOutput(output)) {- if (!output.toolCallId) {- return output.output;- }-- return output.toolName- ? getInAppAgentSilentMcpOutputMessage(- output.toolName,- output.toolCallId,- )- : output.output;+ return toAiSdkToolModelOutput(output);}- return toModelOutput ? toModelOutput(output) : output;+ return toAiSdkToolModelOutput(+ toModelOutput ? toModelOutput(output) : output,+ );};return [toolName, tool];