Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(in-app-agent): improve tool payload and evaluator code rendering (#17044)

Lysander Kiesel5f7da4550d83
diff --git a/web/src/components/design-system/Codeblock/Codeblock.tsx b/web/src/components/design-system/Codeblock/Codeblock.tsx
index efbf46d2b2..3fc91b6c20 100644
--- a/web/src/components/design-system/Codeblock/Codeblock.tsx
+++ b/web/src/components/design-system/Codeblock/Codeblock.tsx
@@ -11,6 +11,7 @@ interface Props {
language: string;
value: string;
theme?: "light" | "dark";
+ borderless?: boolean;
/** Hide the language caption when the surrounding UI already states it. */
showLanguage?: boolean;
/** Match immutable form fields instead of using the recessed code surface. */
@@ -18,7 +19,14 @@ interface Props {
}
const CodeBlock: FC<Props> = memo(
- ({ language, value, theme, showLanguage = true, variant = "default" }) => {
+ ({
+ language,
+ value,
+ theme,
+ borderless = false,
+ showLanguage = true,
+ variant = "default",
+ }) => {
const [isCopied, setIsCopied] = useState(false);
const { resolvedTheme } = useTheme();
const appliedTheme = theme ?? resolvedTheme;
@@ -52,7 +60,8 @@ const CodeBlock: FC<Props> = memo(
return (
<div
className={cn(
- "codeblock relative w-full overflow-hidden rounded border font-sans",
+ "codeblock relative w-full overflow-hidden rounded font-sans",
+ !borderless && "border",
variant === "read-only" ? "bg-muted" : "bg-surface-code",
)}
>
diff --git a/web/src/components/ui/CodeJsonViewer.tsx b/web/src/components/ui/CodeJsonViewer.tsx
index 7bd2232d4a..632b5fd6f3 100644
--- a/web/src/components/ui/CodeJsonViewer.tsx
+++ b/web/src/components/ui/CodeJsonViewer.tsx
@@ -47,14 +47,21 @@ export function JSONView(props: {
controlButtons?: React.ReactNode;
externalJsonCollapsed?: boolean;
onToggleCollapse?: () => void;
+ collapseDepth?: number;
+ /** Skip normalization of prepared payloads to preserve exact source strings and field-marker identities. */
+ preserveStrings?: boolean;
+ customizeNode?: (node: unknown) => React.ReactElement | undefined;
}) {
// some users ingest stringified json nested in json, parse it. Also decode
// \uXXXX escapes (e.g. Japanese ingested with Python ensure_ascii=True) so
// non-ASCII content renders as real characters. Already-decoded strings are
// a no-op (decodeUnicodeEscapesOnly returns early when there is no backslash).
const parsedJson = useMemo(
- () => decodeUnicodeInJson(deepParseJson(props.json)),
- [props.json],
+ () =>
+ props.preserveStrings
+ ? props.json
+ : decodeUnicodeInJson(deepParseJson(props.json)),
+ [props.json, props.preserveStrings],
);
const { resolvedTheme } = useTheme();
const { setIsMarkdownEnabled } = useMarkdownContext();
@@ -67,7 +74,8 @@ export function JSONView(props: {
? 100_000_000 // if null, show all (100M chars)
: (props.collapseStringsAfterLength ?? 500);
- const isCollapsed = props.externalJsonCollapsed ?? internalCollapsed;
+ const isFullyCollapsed = props.externalJsonCollapsed ?? internalCollapsed;
+ const collapsed = isFullyCollapsed ? 1 : (props.collapseDepth ?? false);
const handleOnCopy = (event?: React.MouseEvent<HTMLButtonElement>) => {
if (event) {
@@ -135,8 +143,8 @@ export function JSONView(props: {
src={parsedJson}
theme="github"
dark={resolvedTheme === "dark"}
- collapsed={isCollapsed ? 1 : false}
- collapseObjectsAfterLength={isCollapsed ? 0 : 20}
+ collapsed={collapsed}
+ collapseObjectsAfterLength={isFullyCollapsed ? 0 : 20}
collapseStringsAfterLength={collapseStringsAfterLength}
collapseStringMode="word"
customizeCollapseStringUI={(fullSTring, truncated) =>
@@ -146,12 +154,14 @@ export function JSONView(props: {
""
)
}
- displaySize={isCollapsed ? "collapsed" : "expanded"}
+ displaySize={isFullyCollapsed ? "collapsed" : "expanded"}
matchesURL={true}
// Render previewable media (Langfuse refs, data URIs, media URLs)
// as a hover-to-peek chip instead of the raw string; everything
// else falls through to the default value rendering.
customizeNode={({ node }) => {
+ const customNode = props.customizeNode?.(node);
+ if (customNode !== undefined) return customNode;
const descriptor = classifyMediaValue(node);
return descriptor ? (
<MediaReferenceTag descriptor={descriptor} />
@@ -204,9 +214,9 @@ export function JSONView(props: {
size="icon-xs"
onClick={handleToggleCollapse}
className="hover:bg-border -mr-2"
- title={isCollapsed ? "Expand all" : "Collapse all"}
+ title={isFullyCollapsed ? "Expand all" : "Collapse all"}
>
- {isCollapsed ? (
+ {isFullyCollapsed ? (
<UnfoldVertical className="h-3 w-3" />
) : (
<FoldVertical className="h-3 w-3" />
diff --git a/web/src/features/in-app-agent/components/InAppAgentExecution.clienttest.tsx b/web/src/features/in-app-agent/components/InAppAgentExecution.clienttest.tsx
index 6e55f721a1..35bb883927 100644
--- a/web/src/features/in-app-agent/components/InAppAgentExecution.clienttest.tsx
+++ b/web/src/features/in-app-agent/components/InAppAgentExecution.clienttest.tsx
@@ -13,6 +13,7 @@ import {
} from "@langfuse/shared/in-app-agent";
import { TooltipProvider } from "@/src/components/ui/tooltip";
+import { MarkdownContextProvider } from "@/src/features/theming/useMarkdownContext";
import { ControlledInAppAgentWindow } from "./ControlledInAppAgentWindow";
import { InAppAiAgentProvider, useInAppAiAgent } from "./InAppAiAgentProvider";
import styles from "./InAppAgentWindow.module.css";
@@ -259,17 +260,19 @@ function renderExecutionUi({
includeReopenButton?: boolean;
} = {}) {
return render(
- <InAppAiAgentProvider defaultOpen={defaultOpen}>
- <TooltipProvider>
- {includeReopenButton ? <ReopenAssistantButton /> : null}
- <ControlledInAppAgentWindow
- isExpanded={false}
- onDeleteConversation={vi.fn()}
- onExpandedChange={vi.fn()}
- showCloseButton={false}
- />
- </TooltipProvider>
- </InAppAiAgentProvider>,
+ <MarkdownContextProvider>
+ <InAppAiAgentProvider defaultOpen={defaultOpen}>
+ <TooltipProvider>
+ {includeReopenButton ? <ReopenAssistantButton /> : null}
+ <ControlledInAppAgentWindow
+ isExpanded={false}
+ onDeleteConversation={vi.fn()}
+ onExpandedChange={vi.fn()}
+ showCloseButton={false}
+ />
+ </TooltipProvider>
+ </InAppAiAgentProvider>
+ </MarkdownContextProvider>,
);
}
diff --git a/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx b/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx
index 110e090a93..d7ee5ea74f 100644
--- a/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx
+++ b/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx
@@ -1,7 +1,165 @@
import preview from "../../../../.storybook/preview";
-import { expect, fn, userEvent, within } from "storybook/test";
+import { expect, fn, spyOn, userEvent, within } from "storybook/test";
import { InAppAgentToolCallCard } from "./InAppAgentToolCallCard";
+const toolErrorCountSource = `type ToolCall = {
+ id: string;
+ name: string;
+ arguments: unknown;
+ type: string;
+ index: number;
+};
+
+type EvaluationContext = {
+ observation: {
+ input: any;
+ output: any;
+ metadata: any;
+ toolCalls: ToolCall[];
+ };
+ experiment:
+ | {
+ itemExpectedOutput: any;
+ itemMetadata: any;
+ }
+ | undefined;
+};
+
+type ScoreBase = {
+ name: string;
+ comment?: string;
+ configId?: string | null;
+ metadata?: Record<string, unknown>;
+};
+
+type NumericScore = ScoreBase & {
+ dataType: "NUMERIC";
+ value: number;
+};
+
+type Score = NumericScore;
+
+type EvaluationResult = {
+ scores: Score[];
+};
+
+function parseIfJson(value: unknown): unknown {
+ if (typeof value !== "string") return value;
+ try {
+ return JSON.parse(value);
+ } catch {
+ return value;
+ }
+}
+
+function hasErrorShape(value: unknown): boolean {
+ if (value == null) return false;
+ if (typeof value === "object" && !Array.isArray(value)) {
+ const obj = value as Record<string, unknown>;
+ for (const key of Object.keys(obj)) {
+ const lower = key.toLowerCase();
+ if (lower === "error" || lower === "error_message" || lower === "errormessage") {
+ const v = obj[key];
+ if (v != null && v !== false && v !== "") return true;
+ }
+ if (lower === "status" || lower === "state") {
+ const v = String(obj[key]).toLowerCase();
+ if (v === "error" || v === "failed" || v === "failure") return true;
+ }
+ if (lower === "ok" || lower === "success") {
+ if (obj[key] === false) return true;
+ }
+ }
+ }
+ return false;
+}
+
+function evaluate({
+ observation,
+}: EvaluationContext): EvaluationResult {
+ const output = observation.output;
+ const metadata = observation.metadata;
+
+ let errorCount = 0;
+ const reasons: string[] = [];
+
+ // Signal 1: observation-level error status set by instrumentation
+ const level = (metadata && typeof metadata === "object" ? (metadata as any).level : undefined) as
+ | string
+ | undefined;
+ if (level && String(level).toUpperCase() === "ERROR") {
+ errorCount += 1;
+ reasons.push("metadata.level=ERROR");
+ }
+
+ // Signal 2: error-shaped fields in the parsed tool output
+ const parsedOutput = parseIfJson(output);
+ if (hasErrorShape(parsedOutput)) {
+ errorCount += 1;
+ reasons.push("error field present in tool output");
+ }
+
+ return {
+ scores: [
+ {
+ name: "tool_error_count",
+ value: errorCount,
+ dataType: "NUMERIC",
+ comment:
+ errorCount > 0
+ ? \`Detected \${errorCount} tool error indicator(s): \${reasons.join(", ")}.\`
+ : "No tool error indicators detected.",
+ metadata: {
+ reasons,
+ },
+ },
+ ],
+ };
+}`;
+
+const toolErrorRateSource = toolErrorCountSource
+ .replace('name: "tool_error_count"', 'name: "tool_error_rate"')
+ .replace("value: errorCount,", "value: Math.min(errorCount / 2, 1),");
+
+const toolErrorCountDefinition = {
+ name: "tool_error_count",
+ type: "CODE",
+ description:
+ "Counts tool-call errors on a TOOL observation by checking the observation's level/status and scanning parsed output for common error fields.",
+ sourceCode: `${toolErrorCountSource}\n\nconst healthMarker = "\\u2713";`,
+ sourceCodeLanguage: "TYPESCRIPT",
+};
+
+const approvalToolErrorCountArguments = JSON.stringify(
+ toolErrorCountDefinition,
+);
+const toolErrorCountArguments = JSON.stringify({
+ ...toolErrorCountDefinition,
+ evaluatorId: "tool-error-evaluator",
+ sourceCode: toolErrorRateSource,
+});
+const toolErrorCountResultValue = {
+ id: "tool-error-evaluator",
+ name: "tool_error_count",
+ type: "CODE",
+ description: toolErrorCountDefinition.description,
+ versions: [
+ {
+ version: 1,
+ sourceCode: toolErrorCountSource,
+ sourceCodeLanguage: "TYPESCRIPT",
+ },
+ {
+ version: 2,
+ sourceCode: toolErrorRateSource,
+ sourceCodeLanguage: "TYPESCRIPT",
+ },
+ ],
+};
+const toolErrorCountResult = JSON.stringify({
+ content: [{ type: "text", text: JSON.stringify(toolErrorCountResultValue) }],
+});
+
const meta = preview.meta({
component: InAppAgentToolCallCard,
});
@@ -37,16 +195,193 @@ export const Default = meta.story({
});
export const Error = meta.story({
+ name: "(Test) Error",
args: {
isCompact: true,
tool: {
type: "tool",
name: "langfuse_getTraces",
status: "failed",
- args: JSON.stringify({ limit: 10 }, null, 2),
- error: "Failed to load traces: missing project access.",
+ args: "{ not valid JSON",
+ error: "The tool rejected the request.",
+ },
+ },
+ play: async (context) => {
+ const canvas = await showToolCall(context);
+ await expect(canvas.getByText("{ not valid JSON")).toBeVisible();
+ await expect(
+ canvas.getByText("The tool rejected the request."),
+ ).toBeVisible();
+ },
+});
+
+export const ApprovalRequiredWithCode = meta.story({
+ name: "(Test) Approval required with code",
+ args: {
+ isCompact: true,
+ tool: {
+ type: "tool",
+ name: "langfuse_createEvaluator",
+ status: "running",
+ args: approvalToolErrorCountArguments,
+ approval: { id: "approval-code-1", status: "pending" },
+ },
+ onApproveToolCall: fn(),
+ onAlwaysAllowToolCall: fn(),
+ onRejectToolCall: fn(),
+ },
+ play: async ({ canvasElement, args }) => {
+ const canvas = within(canvasElement);
+ await expect(canvas.getByRole("button", { name: "Approve" })).toBeVisible();
+ await expect(canvas.queryByRole("combobox")).not.toBeInTheDocument();
+ await expect(canvas.queryByText("Details")).not.toBeInTheDocument();
+ await expect(canvas.getByText("sourceCode", { exact: true })).toBeVisible();
+ await userEvent.click(canvas.getByRole("button", { name: "View JSON" }));
+ await expect(
+ canvas.queryByRole("button", { name: "Copy code" }),
+ ).not.toBeInTheDocument();
+ await expect(canvasElement).not.toHaveTextContent("const healthMarker");
+ const copy = spyOn(navigator.clipboard, "writeText").mockResolvedValue();
+ try {
+ const rootCopy = canvasElement.querySelector(
+ ".json-view > .json-view--copy",
+ );
+ if (!rootCopy) {
+ throw new globalThis.Error("Missing JSON root copy control");
+ }
+ await userEvent.click(rootCopy);
+ await expect(JSON.parse(copy.mock.calls[0][0])).toEqual(
+ JSON.parse(args.tool.args),
+ );
+ await userEvent.click(
+ canvas.getByRole("button", { name: "View as code: sourceCode" }),
+ );
+ await userEvent.click(canvas.getByRole("button", { name: "Copy code" }));
+ await expect(copy).toHaveBeenLastCalledWith(
+ toolErrorCountDefinition.sourceCode,
+ );
+ } finally {
+ copy.mockRestore();
+ }
+ },
+});
+
+export const EvaluatorToolCall = meta.story({
+ name: "(Test) Evaluator tool call",
+ args: {
+ isCompact: true,
+ tool: {
+ type: "tool",
+ name: "langfuse_updateEvaluator",
+ status: "succeeded",
+ args: toolErrorCountArguments,
+ result: toolErrorCountResult,
},
},
+ play: async (context) => {
+ const canvas = await showToolCall(context);
+ await expect(canvas.getByText("sourceCode", { exact: true })).toBeVisible();
+ await expect(canvas.getByText("versions[0].sourceCode")).toBeVisible();
+ await expect(
+ canvas.getAllByRole("button", { name: "Copy code" }),
+ ).toHaveLength(2);
+ await userEvent.click(
+ canvas.getAllByRole("button", { name: "View JSON" })[1],
+ );
+ await expect(
+ canvas.getByRole("region", { name: "sourceCode" }),
+ ).toBeVisible();
+ await expect(
+ canvas.queryByRole("region", { name: "versions[0].sourceCode" }),
+ ).not.toBeInTheDocument();
+ await expect(
+ canvas.getByRole("button", {
+ name: "View as code: versions[0].sourceCode",
+ }),
+ ).toBeVisible();
+ const copy = spyOn(navigator.clipboard, "writeText").mockResolvedValue();
+ try {
+ const rootCopy = context.canvasElement.querySelector(
+ ".json-view > .json-view--copy",
+ );
+ if (!rootCopy) {
+ throw new globalThis.Error("Missing JSON root copy control");
+ }
+ await userEvent.click(rootCopy);
+ await expect(JSON.parse(copy.mock.calls[0][0])).toEqual(
+ toolErrorCountResultValue,
+ );
+ await userEvent.click(
+ canvas.getByRole("button", {
+ name: "View as code: versions[1].sourceCode",
+ }),
+ );
+ const selectedCode = within(
+ canvas.getByRole("region", { name: "versions[1].sourceCode" }),
+ );
+ await userEvent.click(
+ selectedCode.getByRole("button", { name: "Copy code" }),
+ );
+ await expect(copy).toHaveBeenLastCalledWith(toolErrorRateSource);
+ await expect(
+ canvas.getAllByRole("button", { name: "Copy code" }),
+ ).toHaveLength(2);
+ } finally {
+ copy.mockRestore();
+ }
+ },
+});
+
+export const ApprovalRequired = meta.story({
+ args: {
+ isCompact: true,
+ tool: {
+ type: "tool",
+ name: "langfuse_upsertDataset",
+ status: "running",
+ args: JSON.stringify(
+ {
+ name: "regression-examples",
+ description: "Examples used for release regression tests",
+ },
+ null,
+ 2,
+ ),
+ approval: {
+ id: "approval-1",
+ status: "pending",
+ },
+ },
+ onApproveToolCall: fn(),
+ onAlwaysAllowToolCall: fn(),
+ onRejectToolCall: fn(),
+ },
+});
+
+export const ApprovalSubmitting = meta.story({
+ args: {
+ isCompact: true,
+ tool: {
+ type: "tool",
+ name: "langfuse_upsertDataset",
+ status: "running",
+ args: JSON.stringify(
+ {
+ name: "regression-examples",
+ description: "Examples used for release regression tests",
+ },
+ null,
+ 2,
+ ),
+ approval: {
+ id: "approval-1",
+ status: "submitting",
+ },
+ },
+ onApproveToolCall: fn(),
+ onAlwaysAllowToolCall: fn(),
+ onRejectToolCall: fn(),
+ },
});
export const SandboxBash = meta.story({
@@ -154,9 +489,11 @@ export const SandboxEdit = meta.story({
},
});
-export const ApprovalRequired = meta.story({
+export const ApprovalDisabled = meta.story({
+ name: "(Test) Approval disabled",
args: {
isCompact: true,
+ isDisabled: true,
tool: {
type: "tool",
name: "langfuse_upsertDataset",
@@ -178,73 +515,92 @@ export const ApprovalRequired = meta.story({
onAlwaysAllowToolCall: fn(),
onRejectToolCall: fn(),
},
+ play: async ({ canvasElement }: { canvasElement: HTMLElement }) => {
+ const canvas = within(canvasElement);
+
+ await expect(
+ canvas.getByRole("button", { name: "Approve" }),
+ ).toBeDisabled();
+ await expect(
+ canvas.getByRole("button", {
+ name: "Always approve for this conversation",
+ }),
+ ).toBeDisabled();
+ await expect(
+ canvas.getByRole("button", { name: "Decline" }),
+ ).toBeDisabled();
+ },
});
-export const ApprovalSubmitting = meta.story({
+export const NestedEvaluatorPayload = meta.story({
+ name: "(Test) Nested evaluator payload",
args: {
isCompact: true,
tool: {
type: "tool",
- name: "langfuse_upsertDataset",
- status: "running",
- args: JSON.stringify(
- {
- name: "regression-examples",
- description: "Examples used for release regression tests",
- },
- null,
- 2,
- ),
- approval: {
- id: "approval-1",
- status: "submitting",
- },
+ name: "langfuse_getEvaluator",
+ status: "succeeded",
+ args: JSON.stringify({ evaluatorId: "example-evaluator" }),
+ result: JSON.stringify({
+ result:
+ '{"type":"CODE","sourceCode":"{}","sourceCodeLanguage":"PYTHON","id":9223372036854775807}',
+ }),
},
- onApproveToolCall: fn(),
- onAlwaysAllowToolCall: fn(),
- onRejectToolCall: fn(),
+ },
+ play: async (context) => {
+ const canvas = await showToolCall(context);
+ const copy = spyOn(navigator.clipboard, "writeText").mockResolvedValue();
+ try {
+ await expect(canvas.getByText("result.sourceCode")).toBeVisible();
+ await userEvent.click(canvas.getByRole("button", { name: "Copy code" }));
+ await expect(copy).toHaveBeenLastCalledWith("{}");
+ await userEvent.click(canvas.getByRole("button", { name: "View JSON" }));
+ copy.mockClear();
+ await userEvent.click(
+ context.canvasElement.querySelectorAll(
+ ".json-view > .json-view--copy",
+ )[1],
+ );
+ await expect(JSON.parse(copy.mock.calls[0][0])).toEqual({
+ result: {
+ type: "CODE",
+ sourceCode: "{}",
+ sourceCodeLanguage: "PYTHON",
+ id: "9223372036854775807",
+ },
+ });
+ } finally {
+ copy.mockRestore();
+ }
},
});
-export const ApprovalDisabled = meta.story({
- name: "(Test) Approval disabled",
+export const UnconfiguredCodePayload = meta.story({
+ name: "(Test) Unconfigured code payload",
args: {
isCompact: true,
- isDisabled: true,
tool: {
type: "tool",
- name: "langfuse_upsertDataset",
- status: "running",
- args: JSON.stringify(
- {
- name: "regression-examples",
- description: "Examples used for release regression tests",
- },
- null,
- 2,
- ),
- approval: {
- id: "approval-1",
- status: "pending",
- },
+ name: "constructor",
+ status: "succeeded",
+ args: JSON.stringify({
+ type: "CODE",
+ sourceCode: "return 42",
+ sourceCodeLanguage: "PYTHON",
+ }),
},
- onApproveToolCall: fn(),
- onAlwaysAllowToolCall: fn(),
- onRejectToolCall: fn(),
},
- play: async ({ canvasElement }: { canvasElement: HTMLElement }) => {
- const canvas = within(canvasElement);
-
+ play: async (context) => {
+ const canvas = await showToolCall(context);
+ await expect(canvas.getByText('"return 42"')).toBeVisible();
await expect(
- canvas.getByRole("button", { name: "Approve" }),
- ).toBeDisabled();
+ canvas.queryByRole("button", { name: "View JSON" }),
+ ).not.toBeInTheDocument();
await expect(
- canvas.getByRole("button", {
- name: "Always approve for this conversation",
- }),
- ).toBeDisabled();
+ canvas.queryByRole("button", { name: /View as code/ }),
+ ).not.toBeInTheDocument();
await expect(
- canvas.getByRole("button", { name: "Decline" }),
- ).toBeDisabled();
+ canvas.queryByRole("button", { name: "Copy code" }),
+ ).not.toBeInTheDocument();
},
});
diff --git a/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx b/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx
index bd14007fb4..0dadcb039e 100644
--- a/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx
+++ b/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx
@@ -279,12 +279,18 @@ function DefaultToolCallDetails({ tool }: { tool: InAppAgentToolCallContent }) {
return (
<div className="flex flex-col gap-2">
<InAppAgentToolPayload
+ toolName={tool.name}
+ kind="arguments"
label="Arguments"
value={tool.args}
variant="default"
/>
{result !== undefined && (
- <InAppAgentToolResultPayload status={tool.status} value={result} />
+ <InAppAgentToolResultPayload
+ toolName={tool.name}
+ status={tool.status}
+ value={result}
+ />
)}
</div>
);
diff --git a/web/src/features/in-app-agent/components/InAppAgentToolPayload.tsx b/web/src/features/in-app-agent/components/InAppAgentToolPayload.tsx
index 1f80eec732..51fab54a1c 100644
--- a/web/src/features/in-app-agent/components/InAppAgentToolPayload.tsx
+++ b/web/src/features/in-app-agent/components/InAppAgentToolPayload.tsx
@@ -1,53 +1,168 @@
"use client";
-import { useMemo } from "react";
+import { useMemo, useState } from "react";
+import { CodeBlock } from "@/src/components/design-system/Codeblock/Codeblock";
+import { JSONView } from "@/src/components/ui/CodeJsonViewer";
import { cn } from "@/src/utils/tailwind";
+import {
+ prepareToolPayload,
+ unwrapMcpTextResult,
+} from "./tool-payload-renderers";
+
+type ParsedToolPayload =
+ | { state: "json"; value: unknown }
+ | { state: "raw"; value: string };
export function InAppAgentToolPayload({
label,
value,
variant,
+ toolName,
+ kind,
}: {
label: string;
+ toolName?: string;
+ kind: "arguments" | "result";
value: string;
variant: "default" | "failed" | "denied";
}) {
- const toolPayload = useMemo(() => {
+ const payload = useMemo<ParsedToolPayload>(() => {
const trimmedValue = value.trim();
if (!trimmedValue) {
- return "{}";
+ return { state: "json", value: {} };
}
try {
- return JSON.stringify(JSON.parse(trimmedValue), null, 2);
+ const parsedValue = JSON.parse(trimmedValue) as unknown;
+
+ return {
+ state: "json",
+ value: unwrapMcpTextResult(parsedValue),
+ };
} catch {
- return value;
+ return { state: "raw", value };
}
}, [value]);
- return (
- <div className="space-y-1">
- <p
- className={cn(
- "text-xs font-bold",
- variant === "default" && "text-muted-foreground",
- variant === "failed" && "text-destructive",
- variant === "denied" && "text-dark-yellow",
- )}
- >
- {label}
- </p>
+ const content =
+ payload.state === "json" ? (
+ <StructuredToolPayload
+ key={toolName}
+ label={label}
+ value={payload.value}
+ toolName={toolName}
+ kind={kind}
+ />
+ ) : (
<pre
className={cn(
- "max-h-64 overflow-auto rounded-md p-2 text-xs whitespace-pre-wrap",
+ "max-h-64 overflow-auto rounded-md p-2 font-mono text-xs whitespace-pre-wrap",
variant === "default" && "bg-muted text-muted-foreground",
variant === "failed" && "bg-destructive/10 text-destructive",
variant === "denied" && "bg-light-yellow text-dark-yellow",
)}
>
- {toolPayload}
+ {payload.value}
</pre>
+ );
+
+ return (
+ <div className="ph-no-capture space-y-1">
+ {payload.state === "raw" ? (
+ <p
+ className={cn(
+ "text-xs font-bold",
+ variant === "default" && "text-muted-foreground",
+ variant === "failed" && "text-destructive",
+ variant === "denied" && "text-dark-yellow",
+ )}
+ >
+ {label}
+ </p>
+ ) : null}
+ {content}
+ </div>
+ );
+}
+
+function StructuredToolPayload({
+ label,
+ value,
+ toolName,
+ kind,
+}: {
+ label: string;
+ value: unknown;
+ toolName: string | undefined;
+ kind: "arguments" | "result";
+}) {
+ const { json, codes, markers } = useMemo(
+ () => prepareToolPayload(value, toolName, kind),
+ [value, toolName, kind],
+ );
+ // undefined selects the first field; null displays the JSON tree.
+ const [selectedPath, setSelectedPath] = useState<string | null>();
+ const code =
+ selectedPath === null
+ ? undefined
+ : (codes.get(selectedPath ?? "") ?? codes.values().next().value);
+
+ if (!code) {
+ return (
+ <div className="flex min-w-0 flex-col gap-2">
+ <p className="text-muted-foreground text-xs font-bold">{label}</p>
+ <JSONView
+ json={json}
+ preserveStrings
+ borderless
+ codeClassName="max-h-64 overflow-auto rounded-md bg-muted p-2"
+ collapseDepth={4}
+ customizeNode={(node) => {
+ const field =
+ typeof node === "object" && node !== null
+ ? markers.get(node)
+ : undefined;
+ return field ? (
+ <button
+ type="button"
+ className="text-primary cursor-pointer underline underline-offset-2"
+ aria-label={`View as code: ${field.path}`}
+ onClick={() => {
+ setSelectedPath(field.path);
+ }}
+ >
+ View as code
+ </button>
+ ) : undefined;
+ }}
+ />
+ </div>
+ );
+ }
+
+ return (
+ <div className="flex min-w-0 flex-col gap-2">
+ <div className="flex items-center justify-between gap-2">
+ <p className="text-muted-foreground text-xs font-bold">{label}</p>
+ <button
+ type="button"
+ className="text-muted-foreground cursor-pointer text-xs underline underline-offset-2"
+ onClick={() => {
+ setSelectedPath(null);
+ }}
+ >
+ View JSON
+ </button>
+ </div>
+ <section aria-label={code.path} className="flex min-w-0 flex-col gap-1">
+ <p className="text-muted-foreground font-mono text-xs break-all">
+ {code.path}
+ </p>
+ <div className="max-h-64 overflow-auto rounded-md border">
+ <CodeBlock language={code.language} value={code.value} borderless />
+ </div>
+ </section>
</div>
);
}
diff --git a/web/src/features/in-app-agent/components/InAppAgentToolResultPayload.tsx b/web/src/features/in-app-agent/components/InAppAgentToolResultPayload.tsx
index e36ad2a2d9..6f3edfa206 100644
--- a/web/src/features/in-app-agent/components/InAppAgentToolResultPayload.tsx
+++ b/web/src/features/in-app-agent/components/InAppAgentToolResultPayload.tsx
@@ -19,14 +19,18 @@ const TOOL_CALL_RESULT_PRESENTATION = {
export function InAppAgentToolResultPayload({
status,
value,
+ toolName,
}: {
status: InAppAgentToolCallContent["status"];
value: string;
+ toolName?: string;
}) {
const presentation = TOOL_CALL_RESULT_PRESENTATION[status];
return (
<InAppAgentToolPayload
+ toolName={toolName}
+ kind="result"
label={presentation.label}
value={value}
variant={presentation.variant}
diff --git a/web/src/features/in-app-agent/components/InAppAgentWindow.clienttest.tsx b/web/src/features/in-app-agent/components/InAppAgentWindow.clienttest.tsx
index db30420466..64f8c94f9f 100644
--- a/web/src/features/in-app-agent/components/InAppAgentWindow.clienttest.tsx
+++ b/web/src/features/in-app-agent/components/InAppAgentWindow.clienttest.tsx
@@ -8,6 +8,7 @@ import {
import { ScanSearch } from "lucide-react";
import { InAppAgentRunStatus } from "@langfuse/shared/in-app-agent";
import { TooltipProvider } from "@/src/components/ui/tooltip";
+import { MarkdownContextProvider } from "@/src/features/theming/useMarkdownContext";
import {
InAppAgentWindow,
type InAppAgentWindowProps,
@@ -116,9 +117,11 @@ function windowElement(
};
return (
- <TooltipProvider>
- <InAppAgentWindow {...props} />
- </TooltipProvider>
+ <MarkdownContextProvider>
+ <TooltipProvider>
+ <InAppAgentWindow {...props} />
+ </TooltipProvider>
+ </MarkdownContextProvider>
);
}
diff --git a/web/src/features/in-app-agent/components/tool-payload-renderers.ts b/web/src/features/in-app-agent/components/tool-payload-renderers.ts
new file mode 100644
index 0000000000..afa05fac0d
--- /dev/null
+++ b/web/src/features/in-app-agent/components/tool-payload-renderers.ts
@@ -0,0 +1,164 @@
+import { deepParseJson } from "@langfuse/shared";
+
+const MAX_PAYLOAD_DEPTH = 6;
+
+type PayloadCode = {
+ language: "python" | "typescript";
+ path: string;
+ value: string;
+};
+
+// Only these tool contracts contain evaluator source fields.
+const CODE_TOOL_PAYLOADS: Record<string, readonly ("arguments" | "result")[]> =
+ {
+ langfuse_createEvaluator: ["arguments", "result"],
+ langfuse_updateEvaluator: ["arguments", "result"],
+ langfuse_getEvaluator: ["result"],
+ langfuse_listEvaluators: ["result"],
+ langfuse_testEvaluator: ["arguments"],
+ };
+
+export function unwrapMcpTextResult(value: unknown) {
+ if (
+ !isRecord(value) ||
+ !Array.isArray(value.content) ||
+ value.content.length !== 1 ||
+ !isRecord(value.content[0]) ||
+ value.content[0].type !== "text" ||
+ typeof value.content[0].text !== "string" ||
+ Object.keys(value).some((key) => key !== "content" && key !== "isError") ||
+ value.isError === true
+ ) {
+ return value;
+ }
+
+ try {
+ return JSON.parse(value.content[0].text) as unknown;
+ } catch {
+ return value;
+ }
+}
+
+export function prepareToolPayload(
+ value: unknown,
+ toolName: string | undefined,
+ kind: "arguments" | "result",
+) {
+ const customCode =
+ !!toolName &&
+ Object.hasOwn(CODE_TOOL_PAYLOADS, toolName) &&
+ (CODE_TOOL_PAYLOADS[toolName]?.includes(kind) ?? false);
+ const codes = new Map<string, PayloadCode>();
+ const markers = new WeakMap<object, PayloadCode>();
+
+ function visit(value: unknown, depth: number, path: string): unknown {
+ const code = codes.get(path);
+ if (code) {
+ // Identity distinguishes equal source strings at different paths. toJSON
+ // preserves the original source in whole-payload and nested-node copies.
+ const marker = { toJSON: () => code.value };
+ markers.set(marker, code);
+ return marker;
+ }
+ if (depth > MAX_PAYLOAD_DEPTH) {
+ return value;
+ }
+ if (typeof value === "string") {
+ const trimmed = value.trimStart();
+ if (
+ value.length > 500_000 ||
+ (trimmed[0] !== "{" && trimmed[0] !== "[")
+ ) {
+ return value;
+ }
+ const nestedValue = deepParseJson(value, { maxDepth: 1 });
+ return nestedValue === value
+ ? value
+ : visit(nestedValue, depth + 1, path);
+ }
+ if (Array.isArray(value)) {
+ return value.map((entry, index) =>
+ visit(entry, depth + 1, appendJsonPath(path, index)),
+ );
+ }
+ if (!isRecord(value)) {
+ return value;
+ }
+ if (customCode) {
+ for (const code of readEvaluatorSourceCodes(value, path, depth)) {
+ codes.set(code.path, code);
+ }
+ }
+ return Object.fromEntries(
+ Object.entries(value).map(([key, entry]) => [
+ key,
+ visit(entry, depth + 1, appendJsonPath(path, key)),
+ ]),
+ );
+ }
+ return { json: visit(value, 0, ""), codes, markers };
+}
+
+function readEvaluatorSourceCodes(
+ value: Record<string, unknown>,
+ path: string,
+ depth: number,
+) {
+ if (value.type !== "CODE") {
+ return [];
+ }
+
+ const directCode = readEvaluatorSourceCodeFields(value, path);
+ let codes = directCode ? [directCode] : [];
+
+ if (depth + 2 <= MAX_PAYLOAD_DEPTH && Array.isArray(value.versions)) {
+ for (const [index, version] of value.versions.entries()) {
+ if (!isRecord(version)) {
+ continue;
+ }
+
+ const versionCode = readEvaluatorSourceCodeFields(
+ version,
+ appendJsonPath(appendJsonPath(path, "versions"), index),
+ );
+ if (versionCode) {
+ codes.push(versionCode);
+ }
+ }
+ }
+
+ return codes;
+}
+
+function readEvaluatorSourceCodeFields(
+ value: Record<string, unknown>,
+ path: string,
+) {
+ if (
+ typeof value.sourceCode !== "string" ||
+ (value.sourceCodeLanguage !== "PYTHON" &&
+ value.sourceCodeLanguage !== "TYPESCRIPT")
+ ) {
+ return null;
+ }
+
+ return {
+ language: value.sourceCodeLanguage === "PYTHON" ? "python" : "typescript",
+ path: appendJsonPath(path, "sourceCode"),
+ value: value.sourceCode,
+ } satisfies PayloadCode;
+}
+
+function appendJsonPath(path: string, key: string | number) {
+ if (typeof key === "number") {
+ return `${path}[${key}]`;
+ }
+ if (!/^[A-Za-z_$][\w$]*$/.test(key)) {
+ return `${path}[${JSON.stringify(key)}]`;
+ }
+ return path ? `${path}.${key}` : key;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}