Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(assistant): Optimize sandbox & silent mcp tool prompt guidance (#16411)

Ben Bachemfa01af7202a4
diff --git a/packages/shared/src/in-app-agent/server/persistence.test.ts b/packages/shared/src/in-app-agent/server/persistence.test.ts
index 72ac566ee3..8b4a699c22 100644
--- a/packages/shared/src/in-app-agent/server/persistence.test.ts
+++ b/packages/shared/src/in-app-agent/server/persistence.test.ts
@@ -2,7 +2,6 @@ import { EventType } from "@ag-ui/core";
import { describe, expect, it } from "vitest";
import type { PrismaClient } from "../../db";
-import { IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE } from "../constants";
import {
createSandboxToolCallFileAccumulator,
getConversationMessages,
@@ -13,6 +12,8 @@ describe("getConversationMessages", () => {
const content = JSON.stringify({
type: "silent-mcp-output",
output: { data: [{ id: "observation-1" }] },
+ toolCallId: "tool-call-1",
+ toolName: "langfuse_getHealth",
});
const prisma = {
inAppAgentEvent: {
@@ -43,7 +44,8 @@ describe("getConversationMessages", () => {
id: "tool-result-1",
role: "tool",
toolCallId: "tool-call-1",
- content: IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE,
+ content:
+ "Output saved to /workspace/tool_calls/langfuse_getHealth_tool-call-1.json",
},
]);
});
@@ -93,7 +95,7 @@ describe("createSandboxToolCallFileAccumulator", () => {
expect(accumulator.getFiles()).toEqual([
{
- path: "tool_calls/2026-08-05T00-00-00.000Z_langfuse_getHealth_tool-call-1.json",
+ path: "tool_calls/langfuse_getHealth_tool-call-1.json",
content: JSON.stringify(
{
request: { projectId: "project-1" },
diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.ts
index 99f4b1d0c6..9d81750099 100644
--- a/packages/shared/src/in-app-agent/server/persistence.ts
+++ b/packages/shared/src/in-app-agent/server/persistence.ts
@@ -40,6 +40,7 @@ import { IN_APP_AGENT_REDIRECT_TOOL_NAME } from "../constants";
import { safeJsonParse } from "../../utils/json";
import {
type CompletedInAppAgentMcpToolCall,
+ getInAppAgentSilentMcpOutputFilePath,
getPublicInAppAgentMcpToolResultContent,
getSandboxInAppAgentMcpToolResultContent,
} from "./toolResults";
@@ -325,7 +326,10 @@ export function createSandboxToolCallFileAccumulator(
}
files.push({
- path: `tool_calls/${formatSandboxToolCallTimestamp(draft?.createdAt ?? toolCall.createdAt)}_${draft?.toolName ?? toolCall.toolName}_${toolCall.toolCallId}.json`,
+ path: getInAppAgentSilentMcpOutputFilePath(
+ draft?.toolName ?? toolCall.toolName,
+ toolCall.toolCallId,
+ ),
content: JSON.stringify(
{
request: draft
@@ -400,7 +404,7 @@ export function createSandboxToolCallFileAccumulator(
}
files.push({
- path: `tool_calls/${formatSandboxToolCallTimestamp(draft.createdAt)}_${draft.toolName}_${toolCallId}.json`,
+ path: getInAppAgentSilentMcpOutputFilePath(draft.toolName, toolCallId),
content: JSON.stringify(
{
request: parseSandboxToolCallValue(draft.request),
@@ -1492,10 +1496,6 @@ function parseSandboxToolCallValue(
return parsed === undefined ? value : parsed;
}
-function formatSandboxToolCallTimestamp(date: Date) {
- return date.toISOString().replaceAll(":", "-");
-}
-
function getDefaultConversationTitle(date: Date) {
const weekday = date.toLocaleDateString("en-US", { weekday: "long" });
const hours = String(date.getHours()).padStart(2, "0");
diff --git a/packages/shared/src/in-app-agent/server/toolResults.ts b/packages/shared/src/in-app-agent/server/toolResults.ts
index 916dd0ba5b..c781e019a1 100644
--- a/packages/shared/src/in-app-agent/server/toolResults.ts
+++ b/packages/shared/src/in-app-agent/server/toolResults.ts
@@ -9,6 +9,8 @@ import type { AgUiEvent } from "../schema";
export type SilentInAppAgentMcpToolOutput = {
type: typeof IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE;
output: unknown;
+ toolCallId?: string;
+ toolName?: string;
};
export type CompletedInAppAgentMcpToolCall = {
@@ -20,11 +22,32 @@ export type CompletedInAppAgentMcpToolCall = {
createdAt: Date;
};
+export const getInAppAgentSilentMcpOutputFilePath = (
+ toolName: string,
+ toolCallId: string,
+) => `tool_calls/${toolName}_${toolCallId}.json`;
+
+export const getInAppAgentSilentMcpOutputMessage = (
+ toolName: string,
+ toolCallId: string,
+) =>
+ `Output saved to /workspace/${getInAppAgentSilentMcpOutputFilePath(toolName, toolCallId)}`;
+
export function getPublicInAppAgentMcpToolResultContent(content: string) {
try {
- return isSilentInAppAgentMcpToolOutput(JSON.parse(content) as unknown)
- ? IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE
- : content;
+ const output = JSON.parse(content) as unknown;
+ if (!isSilentInAppAgentMcpToolOutput(output)) {
+ return content;
+ }
+
+ if (!output.toolCallId || !output.toolName) {
+ return IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE;
+ }
+
+ return getInAppAgentSilentMcpOutputMessage(
+ output.toolName,
+ output.toolCallId,
+ );
} catch {
return content;
}
diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.ts
index b98d863f3e..a8439aed71 100644
--- a/worker/src/features/in-app-agent/runtime/agent.test.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts
@@ -1201,6 +1201,9 @@ describe("createAgUiStream", () => {
).join(", "),
}),
);
+ expect(
+ promptMocks.compile.mock.calls[0]?.[0].sandboxFilesystem,
+ ).not.toContain("tool_calls");
expect(promptMocks.compile).toHaveBeenCalledWith(
expect.objectContaining({
userContext: expect.stringContaining('"user_name": "Ada Lovelace"'),
@@ -1440,8 +1443,11 @@ describe("createAgUiStream", () => {
const agentConfig = vi.mocked(Agent).mock.calls.at(-1)?.[0];
const runInstruction = (agentConfig?.defaultOptions as { system?: string })
?.system;
- expect(runInstruction).toContain("has been replaced with an empty one");
- expect(runInstruction).toContain("restored in full");
+ expect(runInstruction).toContain("has expired and been replaced");
+ expect(runInstruction).toContain(
+ "Persisted tool-output files explicitly named in tool results remain available",
+ );
+ expect(runInstruction).not.toContain("/workspace/tool_calls");
expect(
promptMocks.compile.mock.calls.at(-1)?.[0]?.sandboxFilesystem,
).not.toContain("has been replaced with an empty one");
@@ -1893,6 +1899,8 @@ describe("createAgUiStream", () => {
const input = createToolApprovalResumeInput(true, { silent: true });
adapterEvents.createScoreConfigExecute.mockResolvedValueOnce({
type: "silent-mcp-output",
+ toolCallId: "tool-call-1",
+ toolName: "langfuse_createScoreConfig",
output: {
id: "score-config-1",
name: "readiness",
@@ -1900,7 +1908,8 @@ describe("createAgUiStream", () => {
},
});
adapterEvents.createScoreConfigToModelOutput.mockImplementationOnce(
- async () => "Output saved to /workspace/tool_calls",
+ async () =>
+ "Output saved to /workspace/tool_calls/langfuse_createScoreConfig_tool-call-1.json",
);
adapterEvents.inputs = [];
adapterEvents.items = [
@@ -1950,15 +1959,20 @@ describe("createAgUiStream", () => {
expect(resumedToolMessage).toMatchObject({
role: "tool",
- content: "Output saved to /workspace/tool_calls",
+ content:
+ "Output saved to /workspace/tool_calls/langfuse_createScoreConfig_tool-call-1.json",
});
- expect(streamedText).toContain("Output saved to /workspace/tool_calls");
+ expect(streamedText).toContain(
+ "Output saved to /workspace/tool_calls/langfuse_createScoreConfig_tool-call-1.json",
+ );
expect(streamedText).not.toContain("full-tool-output");
expect(persistedEvents).toContainEqual(
expect.objectContaining({
type: EventType.TOOL_CALL_RESULT,
content: JSON.stringify({
type: "silent-mcp-output",
+ toolCallId: "tool-call-1",
+ toolName: "langfuse_createScoreConfig",
output: {
id: "score-config-1",
name: "readiness",
diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.ts
index 172ef74668..1d56316745 100644
--- a/worker/src/features/in-app-agent/runtime/agent.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.ts
@@ -151,17 +151,16 @@ function formatSandboxContext(sandbox?: InAppAgentSandbox): string {
<sandbox_filesystem>
When working in the sandbox filesystem, assume this layout:
- "/workspace" is the current working directory for normal file operations and shell commands.
-- "/workspace/tool_calls" contains all past tool calls and their outputs, including full results requested with the silent argument. Treat this directory as read-only; any changes to it will be discarded before the next tool call.
</sandbox_filesystem>
`;
}
/** Run-scoped, not part of the managed prompt: it describes one turn's environment. */
const SANDBOX_WORKSPACE_RESET_INSTRUCTION = `<sandbox_workspace_reset>
-The sandbox workspace from earlier turns in this conversation expired and has been replaced with an empty one.
-- Any file you created earlier with write, edit, or bash is gone, along with installed packages and all process state.
-- "/workspace/tool_calls" has been restored in full from the conversation history, so results of earlier successful tool calls are still readable there. Failed tool calls were never stored.
-- Do not assume a path exists because you created it earlier in this conversation. Read it first, and recreate what you still need.
+The sandbox session from earlier turns has expired and been replaced.
+- Files created earlier with write, edit, or bash are gone, along with installed packages and process state.
+- Persisted tool-output files explicitly named in tool results remain available.
+- Do not assume any other path exists; read it first and recreate it if needed.
</sandbox_workspace_reset>`;
const STEP_LIMIT_WRAP_UP_INSTRUCTION =
diff --git a/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts b/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts
index 4304b56d23..d6bab93def 100644
--- a/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts
+++ b/worker/src/features/in-app-agent/runtime/sandbox/sandbox.test.ts
@@ -146,7 +146,7 @@ describe("in-app agent sandbox", () => {
});
});
- it("returns the sandbox tool-call directory for silent MCP output", async () => {
+ it("returns the exact sandbox file for silent MCP output", async () => {
const execute = async (input: { query: string }) => ({
result: input.query,
});
@@ -171,17 +171,18 @@ describe("in-app agent sandbox", () => {
tool.inputSchema.safeParse({ query: "test", silent: true }).success,
).toBe(true);
- const output = await tool.execute?.(
- { query: "test", silent: true },
- {} as never,
- );
+ const output = await tool.execute?.({ query: "test", silent: true }, {
+ agent: { toolCallId: "tool-call-1" },
+ } as never);
expect(output).toEqual({
type: "silent-mcp-output",
output: { result: "test" },
+ toolCallId: "tool-call-1",
+ toolName: "search",
});
expect(tool.toModelOutput?.(output)).toBe(
- "Output saved to /workspace/tool_calls",
+ "Output saved to /workspace/tool_calls/search_tool-call-1.json",
);
});
@@ -227,13 +228,14 @@ describe("in-app agent sandbox", () => {
});
await expect(
- tools.listObservations.execute?.(
- { limit: 10, silent: true },
- {} as never,
- ),
+ tools.listObservations.execute?.({ limit: 10, silent: true }, {
+ agent: { toolCallId: "tool-call-2" },
+ } as never),
).resolves.toEqual({
type: "silent-mcp-output",
output: { data: [] },
+ toolCallId: "tool-call-2",
+ toolName: "listObservations",
});
expect(receivedInput).toEqual({ limit: 10 });
});
diff --git a/worker/src/features/in-app-agent/runtime/tools.ts b/worker/src/features/in-app-agent/runtime/tools.ts
index f91e9128bd..9a293467dd 100644
--- a/worker/src/features/in-app-agent/runtime/tools.ts
+++ b/worker/src/features/in-app-agent/runtime/tools.ts
@@ -36,10 +36,10 @@ import {
InAppAgentSandboxReadArgsSchema,
InAppAgentSandboxWriteArgsSchema,
IN_APP_AGENT_REDIRECT_TOOL_NAME,
- IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE,
IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,
} from "@langfuse/shared/in-app-agent";
import {
+ getInAppAgentSilentMcpOutputMessage,
isSilentInAppAgentMcpToolOutput,
type CompletedInAppAgentMcpToolCall,
type SilentInAppAgentMcpToolOutput,
@@ -205,7 +205,7 @@ export function withOptionalSilentMcpOutput(params: {
// Never silence a failure. A tool that reports its error in the result
// (the MCP `isError` convention) would otherwise be collapsed to a
// pointer at a tool_calls file that is deliberately not written.
- if (failureMessage || !silent) {
+ if (failureMessage || !silent || !toolCallId) {
return result;
}
@@ -213,11 +213,22 @@ export function withOptionalSilentMcpOutput(params: {
return {
type: IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,
output: result,
+ toolCallId,
+ toolName,
} satisfies SilentInAppAgentMcpToolOutput;
};
tool.toModelOutput = (output) => {
if (isSilentInAppAgentMcpToolOutput(output)) {
- return IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE;
+ if (!output.toolCallId) {
+ return output.output;
+ }
+
+ return output.toolName
+ ? getInAppAgentSilentMcpOutputMessage(
+ output.toolName,
+ output.toolCallId,
+ )
+ : output.output;
}
return toModelOutput ? toModelOutput(output) : output;