🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(assistant): Optimize sandbox & silent mcp tool prompt guidance (#16411)
Ben Bachemfa01af7202a4
diff --git a/packages/shared/src/in-app-agent/server/persistence.test.ts b/packages/shared/src/in-app-agent/server/persistence.test.tsindex 72ac566ee3..8b4a699c22 100644--- a/packages/shared/src/in-app-agent/server/persistence.test.ts+++ b/packages/shared/src/in-app-agent/server/persistence.test.ts@@ -2,7 +2,6 @@ import { EventType } from "@ag-ui/core";import { describe, expect, it } from "vitest";import type { PrismaClient } from "../../db";-import { IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE } from "../constants";import {createSandboxToolCallFileAccumulator,getConversationMessages,@@ -13,6 +12,8 @@ describe("getConversationMessages", () => {const content = JSON.stringify({type: "silent-mcp-output",output: { data: [{ id: "observation-1" }] },+ toolCallId: "tool-call-1",+ toolName: "langfuse_getHealth",});const prisma = {inAppAgentEvent: {@@ -43,7 +44,8 @@ describe("getConversationMessages", () => {id: "tool-result-1",role: "tool",toolCallId: "tool-call-1",- content: IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE,+ content:+ "Output saved to /workspace/tool_calls/langfuse_getHealth_tool-call-1.json",},]);});@@ -93,7 +95,7 @@ describe("createSandboxToolCallFileAccumulator", () => {expect(accumulator.getFiles()).toEqual([{- path: "tool_calls/2026-08-05T00-00-00.000Z_langfuse_getHealth_tool-call-1.json",+ path: "tool_calls/langfuse_getHealth_tool-call-1.json",content: JSON.stringify({request: { projectId: "project-1" },diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.tsindex 99f4b1d0c6..9d81750099 100644--- a/packages/shared/src/in-app-agent/server/persistence.ts+++ b/packages/shared/src/in-app-agent/server/persistence.ts@@ -40,6 +40,7 @@ import { IN_APP_AGENT_REDIRECT_TOOL_NAME } from "../constants";import { safeJsonParse } from "../../utils/json";import {type CompletedInAppAgentMcpToolCall,+ getInAppAgentSilentMcpOutputFilePath,getPublicInAppAgentMcpToolResultContent,getSandboxInAppAgentMcpToolResultContent,} from "./toolResults";@@ -325,7 +326,10 @@ export function createSandboxToolCallFileAccumulator(}files.push({- path: `tool_calls/${formatSandboxToolCallTimestamp(draft?.createdAt ?? toolCall.createdAt)}_${draft?.toolName ?? toolCall.toolName}_${toolCall.toolCallId}.json`,+ path: getInAppAgentSilentMcpOutputFilePath(+ draft?.toolName ?? toolCall.toolName,+ toolCall.toolCallId,+ ),content: JSON.stringify({request: draft@@ -400,7 +404,7 @@ export function createSandboxToolCallFileAccumulator(}files.push({- path: `tool_calls/${formatSandboxToolCallTimestamp(draft.createdAt)}_${draft.toolName}_${toolCallId}.json`,+ path: getInAppAgentSilentMcpOutputFilePath(draft.toolName, toolCallId),content: JSON.stringify({request: parseSandboxToolCallValue(draft.request),@@ -1492,10 +1496,6 @@ function parseSandboxToolCallValue(return parsed === undefined ? value : parsed;}-function formatSandboxToolCallTimestamp(date: Date) {- return date.toISOString().replaceAll(":", "-");-}-function getDefaultConversationTitle(date: Date) {const weekday = date.toLocaleDateString("en-US", { weekday: "long" });const hours = String(date.getHours()).padStart(2, "0");diff --git a/packages/shared/src/in-app-agent/server/toolResults.ts b/packages/shared/src/in-app-agent/server/toolResults.tsindex 916dd0ba5b..c781e019a1 100644--- a/packages/shared/src/in-app-agent/server/toolResults.ts+++ b/packages/shared/src/in-app-agent/server/toolResults.ts@@ -9,6 +9,8 @@ import type { AgUiEvent } from "../schema";export type SilentInAppAgentMcpToolOutput = {type: typeof IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE;output: unknown;+ toolCallId?: string;+ toolName?: string;};export type CompletedInAppAgentMcpToolCall = {@@ -20,11 +22,32 @@ export type CompletedInAppAgentMcpToolCall = {createdAt: Date;};+export const getInAppAgentSilentMcpOutputFilePath = (+ toolName: string,+ toolCallId: string,+) => `tool_calls/${toolName}_${toolCallId}.json`;++export const getInAppAgentSilentMcpOutputMessage = (+ toolName: string,+ toolCallId: string,+) =>+ `Output saved to /workspace/${getInAppAgentSilentMcpOutputFilePath(toolName, toolCallId)}`;+export function getPublicInAppAgentMcpToolResultContent(content: string) {try {- return isSilentInAppAgentMcpToolOutput(JSON.parse(content) as unknown)- ? IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE- : content;+ const output = JSON.parse(content) as unknown;+ if (!isSilentInAppAgentMcpToolOutput(output)) {+ return content;+ }++ if (!output.toolCallId || !output.toolName) {+ return IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE;+ }++ return getInAppAgentSilentMcpOutputMessage(+ output.toolName,+ output.toolCallId,+ );} catch {return content;}diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.tsindex b98d863f3e..a8439aed71 100644--- a/worker/src/features/in-app-agent/runtime/agent.test.ts+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts@@ -1201,6 +1201,9 @@ describe("createAgUiStream", () => {).join(", "),}),);+ expect(+ promptMocks.compile.mock.calls[0]?.[0].sandboxFilesystem,+ ).not.toContain("tool_calls");expect(promptMocks.compile).toHaveBeenCalledWith(expect.objectContaining({userContext: expect.stringContaining('"user_name": "Ada Lovelace"'),@@ -1440,8 +1443,11 @@ describe("createAgUiStream", () => {const agentConfig = vi.mocked(Agent).mock.calls.at(-1)?.[0];const runInstruction = (agentConfig?.defaultOptions as { system?: string })?.system;- expect(runInstruction).toContain("has been replaced with an empty one");- expect(runInstruction).toContain("restored in full");+ expect(runInstruction).toContain("has expired and been replaced");+ expect(runInstruction).toContain(+ "Persisted tool-output files explicitly named in tool results remain available",+ );+ expect(runInstruction).not.toContain("/workspace/tool_calls");expect(promptMocks.compile.mock.calls.at(-1)?.[0]?.sandboxFilesystem,).not.toContain("has been replaced with an empty one");@@ -1893,6 +1899,8 @@ describe("createAgUiStream", () => {const input = createToolApprovalResumeInput(true, { silent: true });adapterEvents.createScoreConfigExecute.mockResolvedValueOnce({type: "silent-mcp-output",+ toolCallId: "tool-call-1",+ toolName: "langfuse_createScoreConfig",output: {id: "score-config-1",name: "readiness",@@ -1900,7 +1908,8 @@ describe("createAgUiStream", () => {},});adapterEvents.createScoreConfigToModelOutput.mockImplementationOnce(- async () => "Output saved to /workspace/tool_calls",+ async () =>+ "Output saved to /workspace/tool_calls/langfuse_createScoreConfig_tool-call-1.json",);adapterEvents.inputs = [];adapterEvents.items = [@@ -1950,15 +1959,20 @@ describe("createAgUiStream", () => {expect(resumedToolMessage).toMatchObject({role: "tool",- content: "Output saved to /workspace/tool_calls",+ content:+ "Output saved to /workspace/tool_calls/langfuse_createScoreConfig_tool-call-1.json",});- expect(streamedText).toContain("Output saved to /workspace/tool_calls");+ expect(streamedText).toContain(+ "Output saved to /workspace/tool_calls/langfuse_createScoreConfig_tool-call-1.json",+ );expect(streamedText).not.toContain("full-tool-output");expect(persistedEvents).toContainEqual(expect.objectContaining({type: EventType.TOOL_CALL_RESULT,content: JSON.stringify({type: "silent-mcp-output",+ toolCallId: "tool-call-1",+ toolName: "langfuse_createScoreConfig",output: {id: "score-config-1",name: "readiness",diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.tsindex 172ef74668..1d56316745 100644--- a/worker/src/features/in-app-agent/runtime/agent.ts+++ b/worker/src/features/in-app-agent/runtime/agent.ts@@ -151,17 +151,16 @@ function formatSandboxContext(sandbox?: InAppAgentSandbox): string {<sandbox_filesystem>When working in the sandbox filesystem, assume this layout:- "/workspace" is the current working directory for normal file operations and shell commands.-- "/workspace/tool_calls" contains all past tool calls and their outputs, including full results requested with the silent argument. Treat this directory as read-only; any changes to it will be discarded before the next tool call.</sandbox_filesystem>`;}/** Run-scoped, not part of the managed prompt: it describes one turn's environment. */const SANDBOX_WORKSPACE_RESET_INSTRUCTION = `<sandbox_workspace_reset>-The sandbox workspace from earlier turns in this conversation expired and has been replaced with an empty one.-- Any file you created earlier with write, edit, or bash is gone, along with installed packages and all process state.-- "/workspace/tool_calls" has been restored in full from the conversation history, so results of earlier successful tool calls are still readable there. Failed tool calls were never stored.-- Do not assume a path exists because you created it earlier in this conversation. Read it first, and recreate what you still need.+The sandbox session from earlier turns has expired and been replaced.+- Files created earlier with write, edit, or bash are gone, along with installed packages and process state.+- Persisted tool-output files explicitly named in tool results remain available.+- Do not assume any other path exists; read it first and recreate it if needed.</sandbox_workspace_reset>`;const STEP_LIMIT_WRAP_UP_INSTRUCTION =diff --git a/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts b/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.tsindex 4304b56d23..d6bab93def 100644--- a/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts+++ b/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts@@ -146,7 +146,7 @@ describe("in-app agent sandbox", () => {});});- it("returns the sandbox tool-call directory for silent MCP output", async () => {+ it("returns the exact sandbox file for silent MCP output", async () => {const execute = async (input: { query: string }) => ({result: input.query,});@@ -171,17 +171,18 @@ describe("in-app agent sandbox", () => {tool.inputSchema.safeParse({ query: "test", silent: true }).success,).toBe(true);- const output = await tool.execute?.(- { query: "test", silent: true },- {} as never,- );+ const output = await tool.execute?.({ query: "test", silent: true }, {+ agent: { toolCallId: "tool-call-1" },+ } as never);expect(output).toEqual({type: "silent-mcp-output",output: { result: "test" },+ toolCallId: "tool-call-1",+ toolName: "search",});expect(tool.toModelOutput?.(output)).toBe(- "Output saved to /workspace/tool_calls",+ "Output saved to /workspace/tool_calls/search_tool-call-1.json",);});@@ -227,13 +228,14 @@ describe("in-app agent sandbox", () => {});await expect(- tools.listObservations.execute?.(- { limit: 10, silent: true },- {} as never,- ),+ tools.listObservations.execute?.({ limit: 10, silent: true }, {+ agent: { toolCallId: "tool-call-2" },+ } as never),).resolves.toEqual({type: "silent-mcp-output",output: { data: [] },+ toolCallId: "tool-call-2",+ toolName: "listObservations",});expect(receivedInput).toEqual({ limit: 10 });});diff --git a/worker/src/features/in-app-agent/runtime/tools.ts b/worker/src/features/in-app-agent/runtime/tools.tsindex f91e9128bd..9a293467dd 100644--- a/worker/src/features/in-app-agent/runtime/tools.ts+++ b/worker/src/features/in-app-agent/runtime/tools.ts@@ -36,10 +36,10 @@ import {InAppAgentSandboxReadArgsSchema,InAppAgentSandboxWriteArgsSchema,IN_APP_AGENT_REDIRECT_TOOL_NAME,- IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE,IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,} from "@langfuse/shared/in-app-agent";import {+ getInAppAgentSilentMcpOutputMessage,isSilentInAppAgentMcpToolOutput,type CompletedInAppAgentMcpToolCall,type SilentInAppAgentMcpToolOutput,@@ -205,7 +205,7 @@ export function withOptionalSilentMcpOutput(params: {// Never silence a failure. A tool that reports its error in the result// (the MCP `isError` convention) would otherwise be collapsed to a// pointer at a tool_calls file that is deliberately not written.- if (failureMessage || !silent) {+ if (failureMessage || !silent || !toolCallId) {return result;}@@ -213,11 +213,22 @@ export function withOptionalSilentMcpOutput(params: {return {type: IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,output: result,+ toolCallId,+ toolName,} satisfies SilentInAppAgentMcpToolOutput;};tool.toModelOutput = (output) => {if (isSilentInAppAgentMcpToolOutput(output)) {- return IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE;+ if (!output.toolCallId) {+ return output.output;+ }++ return output.toolName+ ? getInAppAgentSilentMcpOutputMessage(+ output.toolName,+ output.toolCallId,+ )+ : output.output;}return toModelOutput ? toModelOutput(output) : output;