🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(assistant): Improve sandbox tool rendering (#16305)
Ben Bachemcd645f7b2f61
diff --git a/packages/shared/src/in-app-agent/schema.ts b/packages/shared/src/in-app-agent/schema.tsindex 6205159be6..e938013731 100644--- a/packages/shared/src/in-app-agent/schema.ts+++ b/packages/shared/src/in-app-agent/schema.ts@@ -43,6 +43,70 @@ export const InAppAgentRedirectActionToolResultSchema = z.object({href: z.string().min(1),});+export const InAppAgentSandboxToolNameSchema = z.enum([+ "read",+ "write",+ "edit",+ "bash",+]);++export const InAppAgentSandboxReadArgsSchema = z.object({+ path: z.string().min(1),+});++export const InAppAgentSandboxWriteArgsSchema = z.object({+ path: z.string().min(1),+ content: z.string(),+});++export const InAppAgentSandboxEditArgsSchema = z.object({+ path: z.string().min(1),+ oldText: z.string(),+ newText: z.string(),+});++export const InAppAgentSandboxBashArgsSchema = z.object({+ command: z.string().min(1),+ timeoutMs: z.number().int().positive().max(120_000).default(120_000),+});++export const InAppAgentSandboxReadResultSchema = z.object({+ path: z.string(),+ content: z.string().nullable(),+});++export const InAppAgentSandboxWriteResultSchema = z.object({+ path: z.string(),+ bytesWritten: z.number().int().nonnegative(),+});++export const InAppAgentSandboxEditResultSchema = z.object({+ path: z.string(),+ replaced: z.boolean(),+});++export const InAppAgentSandboxBashResultSchema = z.object({+ stdout: z.string(),+ stderr: z.string(),+ exitCode: z.number().int(),+ startedAt: z.string().optional(),+ completedAt: z.string().optional(),+});++export const InAppAgentSandboxToolArgsSchemas = {+ read: InAppAgentSandboxReadArgsSchema,+ write: InAppAgentSandboxWriteArgsSchema,+ edit: InAppAgentSandboxEditArgsSchema,+ bash: InAppAgentSandboxBashArgsSchema,+} as const;++export const InAppAgentSandboxToolResultSchemas = {+ read: InAppAgentSandboxReadResultSchema,+ write: InAppAgentSandboxWriteResultSchema,+ edit: InAppAgentSandboxEditResultSchema,+ bash: InAppAgentSandboxBashResultSchema,+} as const;+const AgUiInputContentSourceSchema = z.discriminatedUnion("type", [z.object({type: z.literal("data"),diff --git a/web/src/features/in-app-agent/components/InAppAgentMessage.tsx b/web/src/features/in-app-agent/components/InAppAgentMessage.tsxindex 888c34686b..337e2c974f 100644--- a/web/src/features/in-app-agent/components/InAppAgentMessage.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentMessage.tsx@@ -47,8 +47,7 @@ import {projectMarkdownToRenderedText,} from "./utils/markdown";import styles from "./InAppAgentMessage.module.css";-import { InAppAgentToolPayload } from "./InAppAgentToolPayload";-import { InAppAgentToolResultPayload } from "./InAppAgentToolResultPayload";+import { InAppAgentToolCallDetails } from "./InAppAgentToolCallDetails";import {getInAppAgentToolDisplayName,type InAppAgentToolCallContent,@@ -810,14 +809,7 @@ function ToolCallDisclosure({</span></summary><div className={cn("mt-1.5 mb-1 ml-3 px-3", isCompact && "px-2.5")}>- <div className="flex flex-col gap-2">- <InAppAgentToolPayload- label="Arguments"- value={tool.args}- variant="default"- />- <InAppAgentToolResultPayload tool={tool} />- </div>+ <InAppAgentToolCallDetails tool={tool} /></div></details>);diff --git a/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx b/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsxindex b7e642badc..ced65ebf0f 100644--- a/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx@@ -1,11 +1,21 @@import preview from "../../../../.storybook/preview";-import { expect, fn, within } from "storybook/test";+import { expect, fn, userEvent, within } from "storybook/test";import { InAppAgentToolCallCard } from "./InAppAgentToolCallCard";const meta = preview.meta({component: InAppAgentToolCallCard,});+const showToolCall = async ({+ canvasElement,+}: {+ canvasElement: HTMLElement;+}) => {+ const canvas = within(canvasElement);+ await userEvent.click(canvas.getByText("Show"));+ return canvas;+};+export const Default = meta.story({args: {isCompact: true,@@ -39,6 +49,107 @@ export const Error = meta.story({},});+export const SandboxBash = meta.story({+ args: {+ isCompact: true,+ tool: {+ type: "tool",+ name: "bash",+ status: "succeeded",+ args: JSON.stringify({ command: "pnpm test -- changed.test.ts" }),+ result: JSON.stringify({+ stdout: "Tests 4 passed (4)\n",+ stderr: "",+ exitCode: 0,+ startedAt: "2026-08-19T08:00:00.000Z",+ completedAt: "2026-08-19T08:00:01.000Z",+ }),+ },+ },+ play: async (context) => {+ const canvas = await showToolCall(context);+ await expect(canvas.getByText("$")).toBeVisible();+ await expect(+ canvas.queryByText("Exited with code 0"),+ ).not.toBeInTheDocument();+ await expect(canvas.queryByText("Arguments")).not.toBeInTheDocument();+ await expect(canvas.queryByText("Result")).not.toBeInTheDocument();+ },+});++export const SandboxRead = meta.story({+ args: {+ isCompact: true,+ tool: {+ type: "tool",+ name: "read",+ status: "succeeded",+ args: JSON.stringify({ path: "src/example.ts" }),+ result: JSON.stringify({+ path: "/workspace/src/example.ts",+ content: "export const answer = 42;\n",+ }),+ },+ },+ play: async (context) => {+ const canvas = await showToolCall(context);+ await expect(canvas.getByText("Read")).toBeVisible();+ await expect(canvas.getByText("export const answer = 42;")).toBeVisible();+ await expect(canvas.queryByText("Result")).not.toBeInTheDocument();+ },+});++export const SandboxWrite = meta.story({+ args: {+ isCompact: true,+ tool: {+ type: "tool",+ name: "write",+ status: "succeeded",+ args: JSON.stringify({+ path: "src/example.ts",+ content: "export const answer = 42;\n",+ }),+ result: JSON.stringify({+ path: "/workspace/src/example.ts",+ bytesWritten: 26,+ }),+ },+ },+ play: async (context) => {+ const canvas = await showToolCall(context);+ await expect(canvas.getByText("Write")).toBeVisible();+ await expect(canvas.queryByText(/Written/)).not.toBeInTheDocument();+ await expect(canvas.queryByText("Result")).not.toBeInTheDocument();+ },+});++export const SandboxEdit = meta.story({+ args: {+ isCompact: true,+ tool: {+ type: "tool",+ name: "edit",+ status: "succeeded",+ args: JSON.stringify({+ path: "src/example.ts",+ oldText: "export const answer = 41;\n",+ newText: "export const answer = 42;\n",+ }),+ result: JSON.stringify({+ path: "/workspace/src/example.ts",+ replaced: true,+ }),+ },+ },+ play: async (context) => {+ const canvas = await showToolCall(context);+ await expect(canvas.getByText("Edit")).toBeVisible();+ await expect(canvas.queryByText("Applied")).not.toBeInTheDocument();+ await expect(canvas.queryByText("Result")).not.toBeInTheDocument();+ },+});+export const ApprovalRequired = meta.story({args: {isCompact: true,diff --git a/web/src/features/in-app-agent/components/InAppAgentToolCallCard.tsx b/web/src/features/in-app-agent/components/InAppAgentToolCallCard.tsxindex cf815c8a4c..9d5505bc1c 100644--- a/web/src/features/in-app-agent/components/InAppAgentToolCallCard.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentToolCallCard.tsx@@ -4,8 +4,7 @@ import { useState } from "react";import { Check, Loader2, Wrench } from "lucide-react";import { Button } from "@/src/components/ui/button";import { cn } from "@/src/utils/tailwind";-import { InAppAgentToolPayload } from "./InAppAgentToolPayload";-import { InAppAgentToolResultPayload } from "./InAppAgentToolResultPayload";+import { InAppAgentToolCallDetails } from "./InAppAgentToolCallDetails";import {getInAppAgentToolDisplayName,type InAppAgentToolCallContent,@@ -74,11 +73,7 @@ export function InAppAgentToolCallCard({</span></div><div className="mt-2 space-y-2">- <InAppAgentToolPayload- label="Arguments"- value={tool.args}- variant="default"- />+ <InAppAgentToolCallDetails tool={tool} /><div className="flex flex-wrap items-center gap-2"><Buttontype="button"@@ -163,12 +158,7 @@ export function InAppAgentToolCallCard({</span></summary><div className="mt-2 space-y-2">- <InAppAgentToolPayload- label="Arguments"- value={tool.args}- variant="default"- />- <InAppAgentToolResultPayload tool={tool} />+ <InAppAgentToolCallDetails tool={tool} /></div></details>)}diff --git a/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx b/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsxnew file mode 100644index 0000000000..fe20334098--- /dev/null+++ b/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx@@ -0,0 +1,312 @@+"use client";++import {+ InAppAgentSandboxBashArgsSchema,+ InAppAgentSandboxBashResultSchema,+ InAppAgentSandboxEditArgsSchema,+ InAppAgentSandboxEditResultSchema,+ InAppAgentSandboxReadArgsSchema,+ InAppAgentSandboxReadResultSchema,+ InAppAgentSandboxToolArgsSchemas,+ InAppAgentSandboxToolNameSchema,+ InAppAgentSandboxToolResultSchemas,+ InAppAgentSandboxWriteArgsSchema,+} from "@langfuse/shared/in-app-agent";+import { type z } from "zod";+import { assertUnreachable } from "@/src/utils/types";+import { InAppAgentToolPayload } from "./InAppAgentToolPayload";+import { InAppAgentToolResultPayload } from "./InAppAgentToolResultPayload";+import type { InAppAgentToolCallContent } from "./utils/utils";++export function InAppAgentToolCallDetails({+ tool,+}: {+ tool: InAppAgentToolCallContent;+}) {+ const toolName = InAppAgentSandboxToolNameSchema.safeParse(tool.name);+ if (!toolName.success) {+ return <DefaultToolCallDetails tool={tool} />;+ }++ const args = parseSandboxPayload(+ tool.args,+ InAppAgentSandboxToolArgsSchemas[toolName.data],+ );+ const result = parseSandboxResult(+ tool.result,+ InAppAgentSandboxToolResultSchemas[toolName.data],+ );+ if (!args || !result.isValid) {+ return <DefaultToolCallDetails tool={tool} />;+ }++ if (toolName.data === "read") {+ return (+ <SandboxToolCallDetails tool={tool}>+ <ReadToolCallDetails+ args={InAppAgentSandboxReadArgsSchema.parse(args)}+ result={+ result.data+ ? InAppAgentSandboxReadResultSchema.parse(result.data)+ : undefined+ }+ />+ </SandboxToolCallDetails>+ );+ }++ if (toolName.data === "write") {+ return (+ <SandboxToolCallDetails tool={tool}>+ <WriteToolCallDetails+ args={InAppAgentSandboxWriteArgsSchema.parse(args)}+ />+ </SandboxToolCallDetails>+ );+ }++ if (toolName.data === "edit") {+ return (+ <SandboxToolCallDetails tool={tool}>+ <EditToolCallDetails+ args={InAppAgentSandboxEditArgsSchema.parse(args)}+ result={+ result.data+ ? InAppAgentSandboxEditResultSchema.parse(result.data)+ : undefined+ }+ />+ </SandboxToolCallDetails>+ );+ }++ if (toolName.data === "bash") {+ return (+ <SandboxToolCallDetails tool={tool}>+ <BashToolCallDetails+ args={InAppAgentSandboxBashArgsSchema.parse(args)}+ result={+ result.data+ ? InAppAgentSandboxBashResultSchema.parse(result.data)+ : undefined+ }+ />+ </SandboxToolCallDetails>+ );+ }++ return assertUnreachable(toolName.data);+}++function SandboxToolCallDetails({+ tool,+ children,+}: {+ tool: InAppAgentToolCallContent;+ children: React.ReactNode;+}) {+ return (+ <div className="flex flex-col gap-2">+ {children}+ <InAppAgentToolResultPayload tool={{ ...tool, result: undefined }} />+ </div>+ );+}++function ReadToolCallDetails({+ args,+ result,+}: {+ args: z.infer<typeof InAppAgentSandboxReadArgsSchema>;+ result: z.infer<typeof InAppAgentSandboxReadResultSchema> | undefined;+}) {+ return (+ <SandboxFileFrame operation="Read" path={args.path}>+ {result ? (+ <pre className="max-h-64 overflow-auto p-2.5 font-mono text-xs leading-5 wrap-break-word whitespace-pre-wrap">+ {result.content === null ? (+ <span className="text-destructive">File not found</span>+ ) : result.content === "" ? (+ <span className="text-muted-foreground italic">Empty file</span>+ ) : (+ result.content+ )}+ </pre>+ ) : null}+ </SandboxFileFrame>+ );+}++function WriteToolCallDetails({+ args,+}: {+ args: z.infer<typeof InAppAgentSandboxWriteArgsSchema>;+}) {+ return (+ <SandboxFileFrame operation="Write" path={args.path}>+ <pre className="max-h-64 overflow-auto p-2.5 font-mono text-xs leading-5 wrap-break-word whitespace-pre-wrap">+ {args.content || (+ <span className="text-muted-foreground italic">Empty file</span>+ )}+ </pre>+ </SandboxFileFrame>+ );+}++function EditToolCallDetails({+ args,+ result,+}: {+ args: z.infer<typeof InAppAgentSandboxEditArgsSchema>;+ result: z.infer<typeof InAppAgentSandboxEditResultSchema> | undefined;+}) {+ return (+ <SandboxFileFrame operation="Edit" path={args.path}>+ <div className="max-h-64 overflow-auto py-1.5 font-mono text-xs leading-5">+ <DiffLines type="removed" value={args.oldText} />+ <DiffLines type="added" value={args.newText} />+ </div>+ {result && !result.replaced ? (+ <SandboxFooter>No matching text found</SandboxFooter>+ ) : null}+ </SandboxFileFrame>+ );+}++function BashToolCallDetails({+ args,+ result,+}: {+ args: z.infer<typeof InAppAgentSandboxBashArgsSchema>;+ result: z.infer<typeof InAppAgentSandboxBashResultSchema> | undefined;+}) {+ return (+ <div className="border-border bg-muted text-foreground max-h-64 overflow-auto rounded-md border p-2.5 font-mono text-xs leading-5 wrap-break-word whitespace-pre-wrap">+ <div>+ <span aria-hidden className="text-muted-foreground select-none">+ ${" "}+ </span>+ <span>{args.command}</span>+ </div>+ {result ? (+ <div>+ {result.stdout}+ {result.stdout && result.stderr && !result.stdout.endsWith("\n")+ ? "\n"+ : null}+ {result.stderr ? (+ <span className="text-destructive">{result.stderr}</span>+ ) : null}+ {!result.stdout && !result.stderr ? (+ <span className="text-muted-foreground italic">No output</span>+ ) : null}+ </div>+ ) : null}+ </div>+ );+}++function SandboxFileFrame({+ operation,+ path,+ children,+}: {+ operation: "Read" | "Write" | "Edit";+ path: string;+ children: React.ReactNode;+}) {+ return (+ <div className="border-border bg-muted text-foreground overflow-hidden rounded-md border">+ <div className="border-border flex min-w-0 items-center gap-2 border-b px-2.5 py-2 text-xs">+ <span className="text-muted-foreground shrink-0">{operation}</span>+ <code className="min-w-0 truncate font-mono" title={path}>+ {path}+ </code>+ </div>+ {children}+ </div>+ );+}++function DiffLines({+ type,+ value,+}: {+ type: "removed" | "added";+ value: string;+}) {+ const lines = value.replace(/\n$/, "").split("\n");++ return lines.map((line, index) => (+ <div+ key={`${type}-${index}`}+ className={+ type === "removed"+ ? "flex bg-red-100/80 text-red-900 dark:bg-red-950/60 dark:text-red-200"+ : "flex bg-green-100/80 text-green-900 dark:bg-green-950/60 dark:text-green-200"+ }+ >+ <span+ aria-hidden+ className={+ type === "removed"+ ? "w-7 shrink-0 border-r border-red-300 text-center text-red-600 select-none dark:border-red-900/70 dark:text-red-400"+ : "w-7 shrink-0 border-r border-green-300 text-center text-green-700 select-none dark:border-green-900/70 dark:text-green-400"+ }+ >+ {type === "removed" ? "-" : "+"}+ </span>+ <span className="min-w-0 px-2 wrap-break-word whitespace-pre-wrap">+ {line || " "}+ </span>+ </div>+ ));+}++function SandboxFooter({ children }: { children: React.ReactNode }) {+ return (+ <div className="border-border text-muted-foreground border-t px-2.5 py-1.5 font-mono text-[0.6875rem]">+ {children}+ </div>+ );+}++function DefaultToolCallDetails({ tool }: { tool: InAppAgentToolCallContent }) {+ return (+ <div className="flex flex-col gap-2">+ <InAppAgentToolPayload+ label="Arguments"+ value={tool.args}+ variant="default"+ />+ <InAppAgentToolResultPayload tool={tool} />+ </div>+ );+}++function parseSandboxPayload<TSchema extends z.ZodType>(+ value: string,+ schema: TSchema,+) {+ try {+ return schema.safeParse(JSON.parse(value) as unknown).data;+ } catch {+ return undefined;+ }+}++function parseSandboxResult<TSchema extends z.ZodType>(+ value: string | undefined,+ schema: TSchema,+) {+ if (value === undefined) {+ return { isValid: true, data: undefined } as const;+ }++ const data = parseSandboxPayload(value, schema);+ if (!data) {+ return { isValid: false, data: undefined } as const;+ }++ return { isValid: true, data } as const;+}diff --git a/worker/src/features/in-app-agent/runtime/sandbox/service.ts b/worker/src/features/in-app-agent/runtime/sandbox/service.tsindex f0e4d4b87a..f8d2bb8b5f 100644--- a/worker/src/features/in-app-agent/runtime/sandbox/service.ts+++ b/worker/src/features/in-app-agent/runtime/sandbox/service.ts@@ -4,6 +4,12 @@ import type {SandboxFile,SandboxProvider,} from "./types";+import {+ InAppAgentSandboxBashResultSchema,+ InAppAgentSandboxEditResultSchema,+ InAppAgentSandboxReadResultSchema,+ InAppAgentSandboxWriteResultSchema,+} from "@langfuse/shared/in-app-agent";import { logger, recordIncrement } from "@langfuse/shared/src/server";export async function createInAppAgentSandbox(params: {@@ -84,23 +90,38 @@ export async function createInAppAgentSandbox(params: {};const createExecutionSandbox = (): InAppAgentSandbox => ({- read: async ({ path }) => (await ensureSession()).read({ path }),+ read: async ({ path }) =>+ InAppAgentSandboxReadResultSchema.parse(+ await (await ensureSession()).read({ path }),+ ),write: async ({ path, content }) =>- (await ensureSession()).write({- path,- content,- }),+ InAppAgentSandboxWriteResultSchema.parse(+ await (+ await ensureSession()+ ).write({+ path,+ content,+ }),+ ),edit: async ({ path, oldText, newText }) =>- (await ensureSession()).edit({- path,- oldText,- newText,- }),+ InAppAgentSandboxEditResultSchema.parse(+ await (+ await ensureSession()+ ).edit({+ path,+ oldText,+ newText,+ }),+ ),bash: async ({ command, timeoutMs }) =>- (await ensureSession()).bash({- command,- timeoutMs,- }),+ InAppAgentSandboxBashResultSchema.parse(+ await (+ await ensureSession()+ ).bash({+ command,+ timeoutMs,+ }),+ ),});return {diff --git a/worker/src/features/in-app-agent/runtime/tools.ts b/worker/src/features/in-app-agent/runtime/tools.tsindex f3be52efda..f91e9128bd 100644--- a/worker/src/features/in-app-agent/runtime/tools.ts+++ b/worker/src/features/in-app-agent/runtime/tools.ts@@ -31,6 +31,10 @@ import {TracingSearchType,} from "@langfuse/shared";import {+ InAppAgentSandboxBashArgsSchema,+ InAppAgentSandboxEditArgsSchema,+ InAppAgentSandboxReadArgsSchema,+ InAppAgentSandboxWriteArgsSchema,IN_APP_AGENT_REDIRECT_TOOL_NAME,IN_APP_AGENT_SILENT_MCP_OUTPUT_MESSAGE,IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,@@ -45,36 +49,26 @@ export function createSandboxTools(sandbox: InAppAgentSandbox) {read: createTool({id: "read",description: "Read a file.",- inputSchema: z.object({ path: z.string().min(1) }),+ inputSchema: InAppAgentSandboxReadArgsSchema,execute: async ({ path }) => sandbox.read({ path }),}),write: createTool({id: "write",description: "Create or overwrite a file.",- inputSchema: z.object({- path: z.string().min(1),- content: z.string(),- }),+ inputSchema: InAppAgentSandboxWriteArgsSchema,execute: async ({ path, content }) => sandbox.write({ path, content }),}),edit: createTool({id: "edit",description: "Replace an exact text span inside a file with new text.",- inputSchema: z.object({- path: z.string().min(1),- oldText: z.string(),- newText: z.string(),- }),+ inputSchema: InAppAgentSandboxEditArgsSchema,execute: async ({ path, oldText, newText }) =>sandbox.edit({ path, oldText, newText }),}),bash: createTool({id: "bash",description: "Run a shell command.",- inputSchema: z.object({- command: z.string().min(1),- timeoutMs: z.number().int().positive().max(120_000).default(120_000),- }),+ inputSchema: InAppAgentSandboxBashArgsSchema,execute: async ({ command, timeoutMs }) =>sandbox.bash({ command, timeoutMs }),}),