🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(in-app-agent): improve tool payload and evaluator code rendering (#17044)
Lysander Kiesel5f7da4550d83
diff --git a/web/src/components/design-system/Codeblock/Codeblock.tsx b/web/src/components/design-system/Codeblock/Codeblock.tsxindex efbf46d2b2..3fc91b6c20 100644--- a/web/src/components/design-system/Codeblock/Codeblock.tsx+++ b/web/src/components/design-system/Codeblock/Codeblock.tsx@@ -11,6 +11,7 @@ interface Props {language: string;value: string;theme?: "light" | "dark";+ borderless?: boolean;/** Hide the language caption when the surrounding UI already states it. */showLanguage?: boolean;/** Match immutable form fields instead of using the recessed code surface. */@@ -18,7 +19,14 @@ interface Props {}const CodeBlock: FC<Props> = memo(- ({ language, value, theme, showLanguage = true, variant = "default" }) => {+ ({+ language,+ value,+ theme,+ borderless = false,+ showLanguage = true,+ variant = "default",+ }) => {const [isCopied, setIsCopied] = useState(false);const { resolvedTheme } = useTheme();const appliedTheme = theme ?? resolvedTheme;@@ -52,7 +60,8 @@ const CodeBlock: FC<Props> = memo(return (<divclassName={cn(- "codeblock relative w-full overflow-hidden rounded border font-sans",+ "codeblock relative w-full overflow-hidden rounded font-sans",+ !borderless && "border",variant === "read-only" ? "bg-muted" : "bg-surface-code",)}>diff --git a/web/src/components/ui/CodeJsonViewer.tsx b/web/src/components/ui/CodeJsonViewer.tsxindex 7bd2232d4a..632b5fd6f3 100644--- a/web/src/components/ui/CodeJsonViewer.tsx+++ b/web/src/components/ui/CodeJsonViewer.tsx@@ -47,14 +47,21 @@ export function JSONView(props: {controlButtons?: React.ReactNode;externalJsonCollapsed?: boolean;onToggleCollapse?: () => void;+ collapseDepth?: number;+ /** Skip normalization of prepared payloads to preserve exact source strings and field-marker identities. */+ preserveStrings?: boolean;+ customizeNode?: (node: unknown) => React.ReactElement | undefined;}) {// some users ingest stringified json nested in json, parse it. Also decode// \uXXXX escapes (e.g. Japanese ingested with Python ensure_ascii=True) so// non-ASCII content renders as real characters. Already-decoded strings are// a no-op (decodeUnicodeEscapesOnly returns early when there is no backslash).const parsedJson = useMemo(- () => decodeUnicodeInJson(deepParseJson(props.json)),- [props.json],+ () =>+ props.preserveStrings+ ? props.json+ : decodeUnicodeInJson(deepParseJson(props.json)),+ [props.json, props.preserveStrings],);const { resolvedTheme } = useTheme();const { setIsMarkdownEnabled } = useMarkdownContext();@@ -67,7 +74,8 @@ export function JSONView(props: {? 100_000_000 // if null, show all (100M chars): (props.collapseStringsAfterLength ?? 500);- const isCollapsed = props.externalJsonCollapsed ?? internalCollapsed;+ const isFullyCollapsed = props.externalJsonCollapsed ?? internalCollapsed;+ const collapsed = isFullyCollapsed ? 1 : (props.collapseDepth ?? false);const handleOnCopy = (event?: React.MouseEvent<HTMLButtonElement>) => {if (event) {@@ -135,8 +143,8 @@ export function JSONView(props: {src={parsedJson}theme="github"dark={resolvedTheme === "dark"}- collapsed={isCollapsed ? 1 : false}- collapseObjectsAfterLength={isCollapsed ? 0 : 20}+ collapsed={collapsed}+ collapseObjectsAfterLength={isFullyCollapsed ? 0 : 20}collapseStringsAfterLength={collapseStringsAfterLength}collapseStringMode="word"customizeCollapseStringUI={(fullSTring, truncated) =>@@ -146,12 +154,14 @@ export function JSONView(props: {"")}- displaySize={isCollapsed ? "collapsed" : "expanded"}+ displaySize={isFullyCollapsed ? "collapsed" : "expanded"}matchesURL={true}// Render previewable media (Langfuse refs, data URIs, media URLs)// as a hover-to-peek chip instead of the raw string; everything// else falls through to the default value rendering.customizeNode={({ node }) => {+ const customNode = props.customizeNode?.(node);+ if (customNode !== undefined) return customNode;const descriptor = classifyMediaValue(node);return descriptor ? (<MediaReferenceTag descriptor={descriptor} />@@ -204,9 +214,9 @@ export function JSONView(props: {size="icon-xs"onClick={handleToggleCollapse}className="hover:bg-border -mr-2"- title={isCollapsed ? "Expand all" : "Collapse all"}+ title={isFullyCollapsed ? "Expand all" : "Collapse all"}>- {isCollapsed ? (+ {isFullyCollapsed ? (<UnfoldVertical className="h-3 w-3" />) : (<FoldVertical className="h-3 w-3" />diff --git a/web/src/features/in-app-agent/components/InAppAgentExecution.clienttest.tsx b/web/src/features/in-app-agent/components/InAppAgentExecution.clienttest.tsxindex 6e55f721a1..35bb883927 100644--- a/web/src/features/in-app-agent/components/InAppAgentExecution.clienttest.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentExecution.clienttest.tsx@@ -13,6 +13,7 @@ import {} from "@langfuse/shared/in-app-agent";import { TooltipProvider } from "@/src/components/ui/tooltip";+import { MarkdownContextProvider } from "@/src/features/theming/useMarkdownContext";import { ControlledInAppAgentWindow } from "./ControlledInAppAgentWindow";import { InAppAiAgentProvider, useInAppAiAgent } from "./InAppAiAgentProvider";import styles from "./InAppAgentWindow.module.css";@@ -259,17 +260,19 @@ function renderExecutionUi({includeReopenButton?: boolean;} = {}) {return render(- <InAppAiAgentProvider defaultOpen={defaultOpen}>- <TooltipProvider>- {includeReopenButton ? <ReopenAssistantButton /> : null}- <ControlledInAppAgentWindow- isExpanded={false}- onDeleteConversation={vi.fn()}- onExpandedChange={vi.fn()}- showCloseButton={false}- />- </TooltipProvider>- </InAppAiAgentProvider>,+ <MarkdownContextProvider>+ <InAppAiAgentProvider defaultOpen={defaultOpen}>+ <TooltipProvider>+ {includeReopenButton ? <ReopenAssistantButton /> : null}+ <ControlledInAppAgentWindow+ isExpanded={false}+ onDeleteConversation={vi.fn()}+ onExpandedChange={vi.fn()}+ showCloseButton={false}+ />+ </TooltipProvider>+ </InAppAiAgentProvider>+ </MarkdownContextProvider>,);}diff --git a/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx b/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsxindex 110e090a93..d7ee5ea74f 100644--- a/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentToolCallCard.stories.tsx@@ -1,7 +1,165 @@import preview from "../../../../.storybook/preview";-import { expect, fn, userEvent, within } from "storybook/test";+import { expect, fn, spyOn, userEvent, within } from "storybook/test";import { InAppAgentToolCallCard } from "./InAppAgentToolCallCard";+const toolErrorCountSource = `type ToolCall = {+ id: string;+ name: string;+ arguments: unknown;+ type: string;+ index: number;+};++type EvaluationContext = {+ observation: {+ input: any;+ output: any;+ metadata: any;+ toolCalls: ToolCall[];+ };+ experiment:+ | {+ itemExpectedOutput: any;+ itemMetadata: any;+ }+ | undefined;+};++type ScoreBase = {+ name: string;+ comment?: string;+ configId?: string | null;+ metadata?: Record<string, unknown>;+};++type NumericScore = ScoreBase & {+ dataType: "NUMERIC";+ value: number;+};++type Score = NumericScore;++type EvaluationResult = {+ scores: Score[];+};++function parseIfJson(value: unknown): unknown {+ if (typeof value !== "string") return value;+ try {+ return JSON.parse(value);+ } catch {+ return value;+ }+}++function hasErrorShape(value: unknown): boolean {+ if (value == null) return false;+ if (typeof value === "object" && !Array.isArray(value)) {+ const obj = value as Record<string, unknown>;+ for (const key of Object.keys(obj)) {+ const lower = key.toLowerCase();+ if (lower === "error" || lower === "error_message" || lower === "errormessage") {+ const v = obj[key];+ if (v != null && v !== false && v !== "") return true;+ }+ if (lower === "status" || lower === "state") {+ const v = String(obj[key]).toLowerCase();+ if (v === "error" || v === "failed" || v === "failure") return true;+ }+ if (lower === "ok" || lower === "success") {+ if (obj[key] === false) return true;+ }+ }+ }+ return false;+}++function evaluate({+ observation,+}: EvaluationContext): EvaluationResult {+ const output = observation.output;+ const metadata = observation.metadata;++ let errorCount = 0;+ const reasons: string[] = [];++ // Signal 1: observation-level error status set by instrumentation+ const level = (metadata && typeof metadata === "object" ? (metadata as any).level : undefined) as+ | string+ | undefined;+ if (level && String(level).toUpperCase() === "ERROR") {+ errorCount += 1;+ reasons.push("metadata.level=ERROR");+ }++ // Signal 2: error-shaped fields in the parsed tool output+ const parsedOutput = parseIfJson(output);+ if (hasErrorShape(parsedOutput)) {+ errorCount += 1;+ reasons.push("error field present in tool output");+ }++ return {+ scores: [+ {+ name: "tool_error_count",+ value: errorCount,+ dataType: "NUMERIC",+ comment:+ errorCount > 0+ ? \`Detected \${errorCount} tool error indicator(s): \${reasons.join(", ")}.\`+ : "No tool error indicators detected.",+ metadata: {+ reasons,+ },+ },+ ],+ };+}`;++const toolErrorRateSource = toolErrorCountSource+ .replace('name: "tool_error_count"', 'name: "tool_error_rate"')+ .replace("value: errorCount,", "value: Math.min(errorCount / 2, 1),");++const toolErrorCountDefinition = {+ name: "tool_error_count",+ type: "CODE",+ description:+ "Counts tool-call errors on a TOOL observation by checking the observation's level/status and scanning parsed output for common error fields.",+ sourceCode: `${toolErrorCountSource}\n\nconst healthMarker = "\\u2713";`,+ sourceCodeLanguage: "TYPESCRIPT",+};++const approvalToolErrorCountArguments = JSON.stringify(+ toolErrorCountDefinition,+);+const toolErrorCountArguments = JSON.stringify({+ ...toolErrorCountDefinition,+ evaluatorId: "tool-error-evaluator",+ sourceCode: toolErrorRateSource,+});+const toolErrorCountResultValue = {+ id: "tool-error-evaluator",+ name: "tool_error_count",+ type: "CODE",+ description: toolErrorCountDefinition.description,+ versions: [+ {+ version: 1,+ sourceCode: toolErrorCountSource,+ sourceCodeLanguage: "TYPESCRIPT",+ },+ {+ version: 2,+ sourceCode: toolErrorRateSource,+ sourceCodeLanguage: "TYPESCRIPT",+ },+ ],+};+const toolErrorCountResult = JSON.stringify({+ content: [{ type: "text", text: JSON.stringify(toolErrorCountResultValue) }],+});+const meta = preview.meta({component: InAppAgentToolCallCard,});@@ -37,16 +195,193 @@ export const Default = meta.story({});export const Error = meta.story({+ name: "(Test) Error",args: {isCompact: true,tool: {type: "tool",name: "langfuse_getTraces",status: "failed",- args: JSON.stringify({ limit: 10 }, null, 2),- error: "Failed to load traces: missing project access.",+ args: "{ not valid JSON",+ error: "The tool rejected the request.",+ },+ },+ play: async (context) => {+ const canvas = await showToolCall(context);+ await expect(canvas.getByText("{ not valid JSON")).toBeVisible();+ await expect(+ canvas.getByText("The tool rejected the request."),+ ).toBeVisible();+ },+});++export const ApprovalRequiredWithCode = meta.story({+ name: "(Test) Approval required with code",+ args: {+ isCompact: true,+ tool: {+ type: "tool",+ name: "langfuse_createEvaluator",+ status: "running",+ args: approvalToolErrorCountArguments,+ approval: { id: "approval-code-1", status: "pending" },+ },+ onApproveToolCall: fn(),+ onAlwaysAllowToolCall: fn(),+ onRejectToolCall: fn(),+ },+ play: async ({ canvasElement, args }) => {+ const canvas = within(canvasElement);+ await expect(canvas.getByRole("button", { name: "Approve" })).toBeVisible();+ await expect(canvas.queryByRole("combobox")).not.toBeInTheDocument();+ await expect(canvas.queryByText("Details")).not.toBeInTheDocument();+ await expect(canvas.getByText("sourceCode", { exact: true })).toBeVisible();+ await userEvent.click(canvas.getByRole("button", { name: "View JSON" }));+ await expect(+ canvas.queryByRole("button", { name: "Copy code" }),+ ).not.toBeInTheDocument();+ await expect(canvasElement).not.toHaveTextContent("const healthMarker");+ const copy = spyOn(navigator.clipboard, "writeText").mockResolvedValue();+ try {+ const rootCopy = canvasElement.querySelector(+ ".json-view > .json-view--copy",+ );+ if (!rootCopy) {+ throw new globalThis.Error("Missing JSON root copy control");+ }+ await userEvent.click(rootCopy);+ await expect(JSON.parse(copy.mock.calls[0][0])).toEqual(+ JSON.parse(args.tool.args),+ );+ await userEvent.click(+ canvas.getByRole("button", { name: "View as code: sourceCode" }),+ );+ await userEvent.click(canvas.getByRole("button", { name: "Copy code" }));+ await expect(copy).toHaveBeenLastCalledWith(+ toolErrorCountDefinition.sourceCode,+ );+ } finally {+ copy.mockRestore();+ }+ },+});++export const EvaluatorToolCall = meta.story({+ name: "(Test) Evaluator tool call",+ args: {+ isCompact: true,+ tool: {+ type: "tool",+ name: "langfuse_updateEvaluator",+ status: "succeeded",+ args: toolErrorCountArguments,+ result: toolErrorCountResult,},},+ play: async (context) => {+ const canvas = await showToolCall(context);+ await expect(canvas.getByText("sourceCode", { exact: true })).toBeVisible();+ await expect(canvas.getByText("versions[0].sourceCode")).toBeVisible();+ await expect(+ canvas.getAllByRole("button", { name: "Copy code" }),+ ).toHaveLength(2);+ await userEvent.click(+ canvas.getAllByRole("button", { name: "View JSON" })[1],+ );+ await expect(+ canvas.getByRole("region", { name: "sourceCode" }),+ ).toBeVisible();+ await expect(+ canvas.queryByRole("region", { name: "versions[0].sourceCode" }),+ ).not.toBeInTheDocument();+ await expect(+ canvas.getByRole("button", {+ name: "View as code: versions[0].sourceCode",+ }),+ ).toBeVisible();+ const copy = spyOn(navigator.clipboard, "writeText").mockResolvedValue();+ try {+ const rootCopy = context.canvasElement.querySelector(+ ".json-view > .json-view--copy",+ );+ if (!rootCopy) {+ throw new globalThis.Error("Missing JSON root copy control");+ }+ await userEvent.click(rootCopy);+ await expect(JSON.parse(copy.mock.calls[0][0])).toEqual(+ toolErrorCountResultValue,+ );+ await userEvent.click(+ canvas.getByRole("button", {+ name: "View as code: versions[1].sourceCode",+ }),+ );+ const selectedCode = within(+ canvas.getByRole("region", { name: "versions[1].sourceCode" }),+ );+ await userEvent.click(+ selectedCode.getByRole("button", { name: "Copy code" }),+ );+ await expect(copy).toHaveBeenLastCalledWith(toolErrorRateSource);+ await expect(+ canvas.getAllByRole("button", { name: "Copy code" }),+ ).toHaveLength(2);+ } finally {+ copy.mockRestore();+ }+ },+});++export const ApprovalRequired = meta.story({+ args: {+ isCompact: true,+ tool: {+ type: "tool",+ name: "langfuse_upsertDataset",+ status: "running",+ args: JSON.stringify(+ {+ name: "regression-examples",+ description: "Examples used for release regression tests",+ },+ null,+ 2,+ ),+ approval: {+ id: "approval-1",+ status: "pending",+ },+ },+ onApproveToolCall: fn(),+ onAlwaysAllowToolCall: fn(),+ onRejectToolCall: fn(),+ },+});++export const ApprovalSubmitting = meta.story({+ args: {+ isCompact: true,+ tool: {+ type: "tool",+ name: "langfuse_upsertDataset",+ status: "running",+ args: JSON.stringify(+ {+ name: "regression-examples",+ description: "Examples used for release regression tests",+ },+ null,+ 2,+ ),+ approval: {+ id: "approval-1",+ status: "submitting",+ },+ },+ onApproveToolCall: fn(),+ onAlwaysAllowToolCall: fn(),+ onRejectToolCall: fn(),+ },});export const SandboxBash = meta.story({@@ -154,9 +489,11 @@ export const SandboxEdit = meta.story({},});-export const ApprovalRequired = meta.story({+export const ApprovalDisabled = meta.story({+ name: "(Test) Approval disabled",args: {isCompact: true,+ isDisabled: true,tool: {type: "tool",name: "langfuse_upsertDataset",@@ -178,73 +515,92 @@ export const ApprovalRequired = meta.story({onAlwaysAllowToolCall: fn(),onRejectToolCall: fn(),},+ play: async ({ canvasElement }: { canvasElement: HTMLElement }) => {+ const canvas = within(canvasElement);++ await expect(+ canvas.getByRole("button", { name: "Approve" }),+ ).toBeDisabled();+ await expect(+ canvas.getByRole("button", {+ name: "Always approve for this conversation",+ }),+ ).toBeDisabled();+ await expect(+ canvas.getByRole("button", { name: "Decline" }),+ ).toBeDisabled();+ },});-export const ApprovalSubmitting = meta.story({+export const NestedEvaluatorPayload = meta.story({+ name: "(Test) Nested evaluator payload",args: {isCompact: true,tool: {type: "tool",- name: "langfuse_upsertDataset",- status: "running",- args: JSON.stringify(- {- name: "regression-examples",- description: "Examples used for release regression tests",- },- null,- 2,- ),- approval: {- id: "approval-1",- status: "submitting",- },+ name: "langfuse_getEvaluator",+ status: "succeeded",+ args: JSON.stringify({ evaluatorId: "example-evaluator" }),+ result: JSON.stringify({+ result:+ '{"type":"CODE","sourceCode":"{}","sourceCodeLanguage":"PYTHON","id":9223372036854775807}',+ }),},- onApproveToolCall: fn(),- onAlwaysAllowToolCall: fn(),- onRejectToolCall: fn(),+ },+ play: async (context) => {+ const canvas = await showToolCall(context);+ const copy = spyOn(navigator.clipboard, "writeText").mockResolvedValue();+ try {+ await expect(canvas.getByText("result.sourceCode")).toBeVisible();+ await userEvent.click(canvas.getByRole("button", { name: "Copy code" }));+ await expect(copy).toHaveBeenLastCalledWith("{}");+ await userEvent.click(canvas.getByRole("button", { name: "View JSON" }));+ copy.mockClear();+ await userEvent.click(+ context.canvasElement.querySelectorAll(+ ".json-view > .json-view--copy",+ )[1],+ );+ await expect(JSON.parse(copy.mock.calls[0][0])).toEqual({+ result: {+ type: "CODE",+ sourceCode: "{}",+ sourceCodeLanguage: "PYTHON",+ id: "9223372036854775807",+ },+ });+ } finally {+ copy.mockRestore();+ }},});-export const ApprovalDisabled = meta.story({- name: "(Test) Approval disabled",+export const UnconfiguredCodePayload = meta.story({+ name: "(Test) Unconfigured code payload",args: {isCompact: true,- isDisabled: true,tool: {type: "tool",- name: "langfuse_upsertDataset",- status: "running",- args: JSON.stringify(- {- name: "regression-examples",- description: "Examples used for release regression tests",- },- null,- 2,- ),- approval: {- id: "approval-1",- status: "pending",- },+ name: "constructor",+ status: "succeeded",+ args: JSON.stringify({+ type: "CODE",+ sourceCode: "return 42",+ sourceCodeLanguage: "PYTHON",+ }),},- onApproveToolCall: fn(),- onAlwaysAllowToolCall: fn(),- onRejectToolCall: fn(),},- play: async ({ canvasElement }: { canvasElement: HTMLElement }) => {- const canvas = within(canvasElement);-+ play: async (context) => {+ const canvas = await showToolCall(context);+ await expect(canvas.getByText('"return 42"')).toBeVisible();await expect(- canvas.getByRole("button", { name: "Approve" }),- ).toBeDisabled();+ canvas.queryByRole("button", { name: "View JSON" }),+ ).not.toBeInTheDocument();await expect(- canvas.getByRole("button", {- name: "Always approve for this conversation",- }),- ).toBeDisabled();+ canvas.queryByRole("button", { name: /View as code/ }),+ ).not.toBeInTheDocument();await expect(- canvas.getByRole("button", { name: "Decline" }),- ).toBeDisabled();+ canvas.queryByRole("button", { name: "Copy code" }),+ ).not.toBeInTheDocument();},});diff --git a/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx b/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsxindex bd14007fb4..0dadcb039e 100644--- a/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentToolCallDetails.tsx@@ -279,12 +279,18 @@ function DefaultToolCallDetails({ tool }: { tool: InAppAgentToolCallContent }) {return (<div className="flex flex-col gap-2"><InAppAgentToolPayload+ toolName={tool.name}+ kind="arguments"label="Arguments"value={tool.args}variant="default"/>{result !== undefined && (- <InAppAgentToolResultPayload status={tool.status} value={result} />+ <InAppAgentToolResultPayload+ toolName={tool.name}+ status={tool.status}+ value={result}+ />)}</div>);diff --git a/web/src/features/in-app-agent/components/InAppAgentToolPayload.tsx b/web/src/features/in-app-agent/components/InAppAgentToolPayload.tsxindex 1f80eec732..51fab54a1c 100644--- a/web/src/features/in-app-agent/components/InAppAgentToolPayload.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentToolPayload.tsx@@ -1,53 +1,168 @@"use client";-import { useMemo } from "react";+import { useMemo, useState } from "react";+import { CodeBlock } from "@/src/components/design-system/Codeblock/Codeblock";+import { JSONView } from "@/src/components/ui/CodeJsonViewer";import { cn } from "@/src/utils/tailwind";+import {+ prepareToolPayload,+ unwrapMcpTextResult,+} from "./tool-payload-renderers";++type ParsedToolPayload =+ | { state: "json"; value: unknown }+ | { state: "raw"; value: string };export function InAppAgentToolPayload({label,value,variant,+ toolName,+ kind,}: {label: string;+ toolName?: string;+ kind: "arguments" | "result";value: string;variant: "default" | "failed" | "denied";}) {- const toolPayload = useMemo(() => {+ const payload = useMemo<ParsedToolPayload>(() => {const trimmedValue = value.trim();if (!trimmedValue) {- return "{}";+ return { state: "json", value: {} };}try {- return JSON.stringify(JSON.parse(trimmedValue), null, 2);+ const parsedValue = JSON.parse(trimmedValue) as unknown;++ return {+ state: "json",+ value: unwrapMcpTextResult(parsedValue),+ };} catch {- return value;+ return { state: "raw", value };}}, [value]);- return (- <div className="space-y-1">- <p- className={cn(- "text-xs font-bold",- variant === "default" && "text-muted-foreground",- variant === "failed" && "text-destructive",- variant === "denied" && "text-dark-yellow",- )}- >- {label}- </p>+ const content =+ payload.state === "json" ? (+ <StructuredToolPayload+ key={toolName}+ label={label}+ value={payload.value}+ toolName={toolName}+ kind={kind}+ />+ ) : (<preclassName={cn(- "max-h-64 overflow-auto rounded-md p-2 text-xs whitespace-pre-wrap",+ "max-h-64 overflow-auto rounded-md p-2 font-mono text-xs whitespace-pre-wrap",variant === "default" && "bg-muted text-muted-foreground",variant === "failed" && "bg-destructive/10 text-destructive",variant === "denied" && "bg-light-yellow text-dark-yellow",)}>- {toolPayload}+ {payload.value}</pre>+ );++ return (+ <div className="ph-no-capture space-y-1">+ {payload.state === "raw" ? (+ <p+ className={cn(+ "text-xs font-bold",+ variant === "default" && "text-muted-foreground",+ variant === "failed" && "text-destructive",+ variant === "denied" && "text-dark-yellow",+ )}+ >+ {label}+ </p>+ ) : null}+ {content}+ </div>+ );+}++function StructuredToolPayload({+ label,+ value,+ toolName,+ kind,+}: {+ label: string;+ value: unknown;+ toolName: string | undefined;+ kind: "arguments" | "result";+}) {+ const { json, codes, markers } = useMemo(+ () => prepareToolPayload(value, toolName, kind),+ [value, toolName, kind],+ );+ // undefined selects the first field; null displays the JSON tree.+ const [selectedPath, setSelectedPath] = useState<string | null>();+ const code =+ selectedPath === null+ ? undefined+ : (codes.get(selectedPath ?? "") ?? codes.values().next().value);++ if (!code) {+ return (+ <div className="flex min-w-0 flex-col gap-2">+ <p className="text-muted-foreground text-xs font-bold">{label}</p>+ <JSONView+ json={json}+ preserveStrings+ borderless+ codeClassName="max-h-64 overflow-auto rounded-md bg-muted p-2"+ collapseDepth={4}+ customizeNode={(node) => {+ const field =+ typeof node === "object" && node !== null+ ? markers.get(node)+ : undefined;+ return field ? (+ <button+ type="button"+ className="text-primary cursor-pointer underline underline-offset-2"+ aria-label={`View as code: ${field.path}`}+ onClick={() => {+ setSelectedPath(field.path);+ }}+ >+ View as code+ </button>+ ) : undefined;+ }}+ />+ </div>+ );+ }++ return (+ <div className="flex min-w-0 flex-col gap-2">+ <div className="flex items-center justify-between gap-2">+ <p className="text-muted-foreground text-xs font-bold">{label}</p>+ <button+ type="button"+ className="text-muted-foreground cursor-pointer text-xs underline underline-offset-2"+ onClick={() => {+ setSelectedPath(null);+ }}+ >+ View JSON+ </button>+ </div>+ <section aria-label={code.path} className="flex min-w-0 flex-col gap-1">+ <p className="text-muted-foreground font-mono text-xs break-all">+ {code.path}+ </p>+ <div className="max-h-64 overflow-auto rounded-md border">+ <CodeBlock language={code.language} value={code.value} borderless />+ </div>+ </section></div>);}diff --git a/web/src/features/in-app-agent/components/InAppAgentToolResultPayload.tsx b/web/src/features/in-app-agent/components/InAppAgentToolResultPayload.tsxindex e36ad2a2d9..6f3edfa206 100644--- a/web/src/features/in-app-agent/components/InAppAgentToolResultPayload.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentToolResultPayload.tsx@@ -19,14 +19,18 @@ const TOOL_CALL_RESULT_PRESENTATION = {export function InAppAgentToolResultPayload({status,value,+ toolName,}: {status: InAppAgentToolCallContent["status"];value: string;+ toolName?: string;}) {const presentation = TOOL_CALL_RESULT_PRESENTATION[status];return (<InAppAgentToolPayload+ toolName={toolName}+ kind="result"label={presentation.label}value={value}variant={presentation.variant}diff --git a/web/src/features/in-app-agent/components/InAppAgentWindow.clienttest.tsx b/web/src/features/in-app-agent/components/InAppAgentWindow.clienttest.tsxindex db30420466..64f8c94f9f 100644--- a/web/src/features/in-app-agent/components/InAppAgentWindow.clienttest.tsx+++ b/web/src/features/in-app-agent/components/InAppAgentWindow.clienttest.tsx@@ -8,6 +8,7 @@ import {import { ScanSearch } from "lucide-react";import { InAppAgentRunStatus } from "@langfuse/shared/in-app-agent";import { TooltipProvider } from "@/src/components/ui/tooltip";+import { MarkdownContextProvider } from "@/src/features/theming/useMarkdownContext";import {InAppAgentWindow,type InAppAgentWindowProps,@@ -116,9 +117,11 @@ function windowElement(};return (- <TooltipProvider>- <InAppAgentWindow {...props} />- </TooltipProvider>+ <MarkdownContextProvider>+ <TooltipProvider>+ <InAppAgentWindow {...props} />+ </TooltipProvider>+ </MarkdownContextProvider>);}diff --git a/web/src/features/in-app-agent/components/tool-payload-renderers.ts b/web/src/features/in-app-agent/components/tool-payload-renderers.tsnew file mode 100644index 0000000000..afa05fac0d--- /dev/null+++ b/web/src/features/in-app-agent/components/tool-payload-renderers.ts@@ -0,0 +1,164 @@+import { deepParseJson } from "@langfuse/shared";++const MAX_PAYLOAD_DEPTH = 6;++type PayloadCode = {+ language: "python" | "typescript";+ path: string;+ value: string;+};++// Only these tool contracts contain evaluator source fields.+const CODE_TOOL_PAYLOADS: Record<string, readonly ("arguments" | "result")[]> =+ {+ langfuse_createEvaluator: ["arguments", "result"],+ langfuse_updateEvaluator: ["arguments", "result"],+ langfuse_getEvaluator: ["result"],+ langfuse_listEvaluators: ["result"],+ langfuse_testEvaluator: ["arguments"],+ };++export function unwrapMcpTextResult(value: unknown) {+ if (+ !isRecord(value) ||+ !Array.isArray(value.content) ||+ value.content.length !== 1 ||+ !isRecord(value.content[0]) ||+ value.content[0].type !== "text" ||+ typeof value.content[0].text !== "string" ||+ Object.keys(value).some((key) => key !== "content" && key !== "isError") ||+ value.isError === true+ ) {+ return value;+ }++ try {+ return JSON.parse(value.content[0].text) as unknown;+ } catch {+ return value;+ }+}++export function prepareToolPayload(+ value: unknown,+ toolName: string | undefined,+ kind: "arguments" | "result",+) {+ const customCode =+ !!toolName &&+ Object.hasOwn(CODE_TOOL_PAYLOADS, toolName) &&+ (CODE_TOOL_PAYLOADS[toolName]?.includes(kind) ?? false);+ const codes = new Map<string, PayloadCode>();+ const markers = new WeakMap<object, PayloadCode>();++ function visit(value: unknown, depth: number, path: string): unknown {+ const code = codes.get(path);+ if (code) {+ // Identity distinguishes equal source strings at different paths. toJSON+ // preserves the original source in whole-payload and nested-node copies.+ const marker = { toJSON: () => code.value };+ markers.set(marker, code);+ return marker;+ }+ if (depth > MAX_PAYLOAD_DEPTH) {+ return value;+ }+ if (typeof value === "string") {+ const trimmed = value.trimStart();+ if (+ value.length > 500_000 ||+ (trimmed[0] !== "{" && trimmed[0] !== "[")+ ) {+ return value;+ }+ const nestedValue = deepParseJson(value, { maxDepth: 1 });+ return nestedValue === value+ ? value+ : visit(nestedValue, depth + 1, path);+ }+ if (Array.isArray(value)) {+ return value.map((entry, index) =>+ visit(entry, depth + 1, appendJsonPath(path, index)),+ );+ }+ if (!isRecord(value)) {+ return value;+ }+ if (customCode) {+ for (const code of readEvaluatorSourceCodes(value, path, depth)) {+ codes.set(code.path, code);+ }+ }+ return Object.fromEntries(+ Object.entries(value).map(([key, entry]) => [+ key,+ visit(entry, depth + 1, appendJsonPath(path, key)),+ ]),+ );+ }+ return { json: visit(value, 0, ""), codes, markers };+}++function readEvaluatorSourceCodes(+ value: Record<string, unknown>,+ path: string,+ depth: number,+) {+ if (value.type !== "CODE") {+ return [];+ }++ const directCode = readEvaluatorSourceCodeFields(value, path);+ let codes = directCode ? [directCode] : [];++ if (depth + 2 <= MAX_PAYLOAD_DEPTH && Array.isArray(value.versions)) {+ for (const [index, version] of value.versions.entries()) {+ if (!isRecord(version)) {+ continue;+ }++ const versionCode = readEvaluatorSourceCodeFields(+ version,+ appendJsonPath(appendJsonPath(path, "versions"), index),+ );+ if (versionCode) {+ codes.push(versionCode);+ }+ }+ }++ return codes;+}++function readEvaluatorSourceCodeFields(+ value: Record<string, unknown>,+ path: string,+) {+ if (+ typeof value.sourceCode !== "string" ||+ (value.sourceCodeLanguage !== "PYTHON" &&+ value.sourceCodeLanguage !== "TYPESCRIPT")+ ) {+ return null;+ }++ return {+ language: value.sourceCodeLanguage === "PYTHON" ? "python" : "typescript",+ path: appendJsonPath(path, "sourceCode"),+ value: value.sourceCode,+ } satisfies PayloadCode;+}++function appendJsonPath(path: string, key: string | number) {+ if (typeof key === "number") {+ return `${path}[${key}]`;+ }+ if (!/^[A-Za-z_$][\w$]*$/.test(key)) {+ return `${path}[${JSON.stringify(key)}]`;+ }+ return path ? `${path}.${key}` : key;+}++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}