TypeScript
fix(tools): show images from MCP and OpenCode outputs (#30389)
Yash Singhd51bc58f5e05
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex e4c45728c23..dc12d15d178 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -272,12 +272,18 @@ jobs:env:TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }}run: |+ set -o pipefailfilters=(--filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web')if [[ -n "$TEST_PACKAGES" ]]; thenfilters=()for package in $TEST_PACKAGES; do filters+=(--filter "$package"); donefi- vp run --parallel --concurrency-limit 4 "${filters[@]}" test+ # The runner's trace writer can crash on surrogate pairs in test output.+ vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c '+ import sys+ for line in sys.stdin:+ sys.stdout.write(line.encode("ascii", "backslashreplace").decode("ascii"))+ '# The web suite is the largest outside the server. In the shared Test job it# started last and ran alone for about 40s after everything else finished.diff --git a/apps/mobile/src/features/threads/thread-work-log.tsx b/apps/mobile/src/features/threads/thread-work-log.tsxindex 0ff0271f44f..ae943c81614 100644--- a/apps/mobile/src/features/threads/thread-work-log.tsx+++ b/apps/mobile/src/features/threads/thread-work-log.tsx@@ -69,6 +69,7 @@ import {workEntryRowLabel,} from "../../lib/threadActivity";import {+ isJsonToolText,toolCallLines,turnItemOutputImages,turnItemOutputText,@@ -874,14 +875,16 @@ function WorkspacePreparationRetryButton(props: {);}-function ReplCodeText({+function ToolCodeText({code,+ language,theme,}: {readonly code: string;+ readonly language: "typescript" | "html" | "json";readonly theme: "light" | "dark";}) {- const lines = useMarkdownCodeHighlight({ code, language: "typescript", theme, enabled: true });+ const lines = useMarkdownCodeHighlight({ code, language, theme, enabled: true });let offset = 0;const highlighted = lines?.flatMap((line, lineIndex) => {const separator = lineIndex > 0 ? "\n" : "";@@ -1255,20 +1258,32 @@ const ThreadWorkLogRow = function ThreadWorkLogRow() : call ? ([call.command,- ...(call.args ?? []).map(([key, value]) =>- key === "code" &&- shownItem.type === "dynamic_tool" &&- isJavaScriptReplTool(shownItem.toolName) ? (+ ...(call.args ?? []).map(([key, value]) => {+ const language =+ key === "html"+ ? "html"+ : key === "code" &&+ shownItem.type === "dynamic_tool" &&+ isJavaScriptReplTool(shownItem.toolName)+ ? "typescript"+ : isJsonToolText(value)+ ? "json"+ : null;+ return language ? (<View key={key}><Text className="font-mono text-2xs leading-normal text-foreground-muted">{key}</Text>- <ReplCodeText code={value} theme={props.themeAppearance} />+ <ToolCodeText+ code={value}+ language={language}+ theme={props.themeAppearance}+ /></View>) : (`${key} ${value}`- ),- ),+ );+ }),call.argsText,].filter((line) => Boolean(line))@@ -1281,7 +1296,11 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(selectableclassName="font-mono text-2xs leading-normal text-foreground">- {line}+ {isJsonToolText(line) ? (+ <ToolCodeText code={line} language="json" theme={props.themeAppearance} />+ ) : (+ line+ )}</Text>) : (line@@ -1289,7 +1308,11 @@ const ThreadWorkLogRow = function ThreadWorkLogRow()) : fullDetail ? (<Text selectable className="font-mono text-2xs leading-normal text-foreground-muted">- {fullDetail}+ {isJsonToolText(fullDetail) ? (+ <ToolCodeText code={fullDetail} language="json" theme={props.themeAppearance} />+ ) : (+ fullDetail+ )}</Text>) : null}{fetchedOutput ? (@@ -1300,7 +1323,15 @@ const ThreadWorkLogRow = function ThreadWorkLogRow((!call || call.command || call.args || call.argsText) && "mt-1.5",)}>- {fetchedOutput}+ {isJsonToolText(fetchedOutput) ? (+ <ToolCodeText+ code={fetchedOutput}+ language="json"+ theme={props.themeAppearance}+ />+ ) : (+ fetchedOutput+ )}</Text>) : null}{streamsOutput ? null : exitLine}diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.tsindex ef721644f6a..3f771d6abee 100644--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts@@ -26,6 +26,7 @@ import type {} from "@anthropic-ai/claude-agent-sdk/sdk-tools";import * as NodeCrypto from "@effect/platform-node/NodeCrypto";import * as NodeServices from "@effect/platform-node/NodeServices";+import { toolOutputImages } from "@t3tools/shared/toolOutput";import {ChatAttachmentId,type ChatAttachment,@@ -4844,11 +4845,19 @@ describe("ClaudeAdapterV2 background wake turns", () => {{type: "tool_result",tool_use_id: id,- content: "Weather result",+ content: [+ { type: "text", text: "Weather result" },+ {+ type: "image",+ source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },+ },+ ],is_error: isError,},],},+ tool_use_result:+ title === undefined ? { screenshot: { mimeType: "image/png" } } : undefined,}),);yield* Queue.offer(@@ -4881,6 +4890,9 @@ describe("ClaudeAdapterV2 background wake turns", () => {assert.deepEqual(item.input, { city: "Berlin" });}assert.equal(new Set(items.map((item) => item.id)).size, 1);+ assert.deepEqual(toolOutputImages(items.at(-1)?.output), [+ { mimeType: "image/png", data: "iVBORw0KGgo=" },+ ]);}).pipe(Effect.scoped,Effect.provide(diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.tsindex d2ae14545d9..98c920f5053 100644--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts@@ -1,4 +1,5 @@import * as DocumentText from "../../provider/DocumentText.ts";+import { mcpToolResultOutput, toolOutputImages } from "@t3tools/shared/toolOutput";import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts";import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts";import {@@ -3084,7 +3085,14 @@ function claudeNativeToolOutputValue(output: ClaudeNativeToolOutput): unknown |case "none":return undefined;case "content_block":+ return output.value;case "structured_tool_use_result":+ if (toolOutputImages(output.value).length === 0) {+ return mcpToolResultOutput({+ content: output.fallbackValue,+ structuredContent: output.value,+ });+ }return output.value;default:return assertNever(output);diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.tsindex 69fc35aa39b..0f8e0ef1f33 100644--- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts+++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts@@ -50,6 +50,7 @@ import {import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model";import { appendGeneratedImageEmbeds } from "@t3tools/shared/generatedImageMarkdown";import { resolveSpawnCommand } from "@t3tools/shared/shell";+import { mcpToolResultOutput } from "@t3tools/shared/toolOutput";import type {ChatAttachment,OrchestrationV2AppThread,@@ -490,9 +491,7 @@ function codexMcpToolOutput(const resultOutput =item.result === null || item.result === undefined? undefined- : item.result.structuredContent !== null && item.result.structuredContent !== undefined- ? item.result.structuredContent- : item.result.content;+ : mcpToolResultOutput(item.result);if (item.error === null || item.error === undefined) {return resultOutput;diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.tsindex f9819070fd8..e4e0851e82c 100644--- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts+++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts@@ -1,4 +1,5 @@import * as NodeURL from "node:url";+import { toolOutputImages } from "@t3tools/shared/toolOutput";import * as NodeCrypto from "@effect/platform-node/NodeCrypto";import * as NodeServices from "@effect/platform-node/NodeServices";import { assert, it } from "@effect/vitest";@@ -2715,48 +2716,67 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {}).pipe(Effect.scoped),);- it.effect("keeps thread-scoped T3 waits as dynamic tools through their lifecycle", () =>- Effect.gen(function* () {- const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };- const name = "t3-code-web_delegate_task";- const input = { task: "Review", mode: "wait" };- const { runtime, thread } = yield* resumed([- out("session.prompt", { sessionID: SESSION, text: "<any>" }),- promptAccepted,- event("session.tool.input.started", { ...tool, name }),- event("session.tool.called", { ...tool, name, input, executed: true }),- event("session.tool.success", {- ...tool,- content: [{ type: "text", text: "Reviewed" }],- metadata: {},- executed: true,- }),- event("session.execution.succeeded", { sessionID: SESSION }),- ]);- const events = yield* runtime.events.pipe(- Stream.takeUntil((event) => event.type === "turn.terminal"),- Stream.runCollect,- Effect.forkScoped,- );- yield* runtime.startTurn(turnInput(thread));- const items = (yield* Fiber.join(events)).flatMap((event) =>- event.type === "turn_item.updated" &&- event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)- ? [event.turnItem]- : [],- );- assert.deepEqual(- items.map((item) => item.type),- ["dynamic_tool", "dynamic_tool", "dynamic_tool"],- );- assert.deepEqual(- items.map((item) => item.status),- ["running", "running", "completed"],- );- const last = items.at(-1);- assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);- assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);- }).pipe(Effect.scoped),+ it.effect.each(["delegate_task", "html_preview", "mcp__acme__edit_document"])(+ "keeps MCP %s as a dynamic tool through its lifecycle",+ (toolName) =>+ Effect.gen(function* () {+ const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };+ const name = toolName.startsWith("mcp__") ? toolName : `t3-code-web_${toolName}`;+ const input =+ toolName !== "delegate_task"+ ? { html: "<p>Preview</p>" }+ : { task: "Review", mode: "wait" };+ const { runtime, thread } = yield* resumed([+ out("session.prompt", { sessionID: SESSION, text: "<any>" }),+ promptAccepted,+ event("session.tool.input.started", { ...tool, name }),+ event("session.tool.called", { ...tool, name, input, executed: true }),+ event(+ "session.tool.success",+ {+ ...tool,+ content: [+ { type: "text", text: "Reviewed" },+ ...(toolName !== "delegate_task"+ ? [{ type: "file", uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" }]+ : []),+ ],+ metadata: {},+ executed: true,+ },+ undefined,+ 2,+ ),+ event("session.execution.succeeded", { sessionID: SESSION }),+ ]);+ const events = yield* runtime.events.pipe(+ Stream.takeUntil((event) => event.type === "turn.terminal"),+ Stream.runCollect,+ Effect.forkScoped,+ );+ yield* runtime.startTurn(turnInput(thread));+ const items = (yield* Fiber.join(events)).flatMap((event) =>+ event.type === "turn_item.updated" &&+ event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)+ ? [event.turnItem]+ : [],+ );+ assert.deepEqual(+ items.map((item) => item.type),+ ["dynamic_tool", "dynamic_tool", "dynamic_tool"],+ );+ assert.deepEqual(+ items.map((item) => item.status),+ ["running", "running", "completed"],+ );+ const last = items.at(-1);+ assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);+ assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);+ assert.deepEqual(+ toolOutputImages(last?.type === "dynamic_tool" ? last.output : null),+ toolName !== "delegate_task" ? [{ mimeType: "image/png", data: "iVBORw0KGgo=" }] : [],+ );+ }).pipe(Effect.scoped),);it.effect("stores a shell command that exits non-zero as failed", () =>diff --git a/apps/web/src/components/chat/V2ItemInspector.tsx b/apps/web/src/components/chat/V2ItemInspector.tsxindex ee6072b926b..8d314a1bd1b 100644--- a/apps/web/src/components/chat/V2ItemInspector.tsx+++ b/apps/web/src/components/chat/V2ItemInspector.tsx@@ -1,6 +1,7 @@import type { ToolCallDiffTarget } from "../../diffPanelStore";import type { EnvironmentId, OrchestrationV2ProjectedTurnItem } from "@t3tools/contracts";import {+ isJsonToolText,toolCallLines,turnItemDetailRevision,turnItemNeedsDetailFetch,@@ -43,7 +44,7 @@ function SyntaxTokens({language,}: {readonly text: string;- readonly language: "json" | "typescript";+ readonly language: "json" | "typescript" | "html";}) {const resolvedTheme = useRenderedAppearance();const highlighter = use(getSyntaxHighlighterPromise(language));@@ -64,16 +65,6 @@ function SyntaxTokens({const monoClassName ="font-mono text-(length:--font-size-code,var(--text-2xs)) leading-relaxed whitespace-pre-wrap break-words select-text";-function isJsonText(text: string): boolean {- if (!text) return false;- try {- JSON.parse(text);- return true;- } catch {- return false;- }-}-function StructuredValue({value,highlightJson = false,@@ -81,10 +72,10 @@ function StructuredValue({}: {readonly value: unknown;readonly highlightJson?: boolean;- readonly language?: "typescript";+ readonly language?: "typescript" | "html" | "json";}) {const text = typeof value === "string" ? value : JSON.stringify(value, null, 2);- const isJson = highlightJson && isJsonText(text);+ const isJson = highlightJson && isJsonToolText(text);if (!text) return null;return (<pre className={cn("max-h-80 overflow-auto text-muted-foreground", monoClassName)}>@@ -150,8 +141,8 @@ interface ToolOutputState {readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;}-function ToolOutput(props: ToolOutputState) {- const images = props.images.map((resource) => (+function ToolOutputImages(props: ToolOutputState) {+ return props.images.map((resource) => (<ChatMarkdownAssetImagekey={resource.index}environmentId={props.environmentId}@@ -161,21 +152,18 @@ function ToolOutput(props: ToolOutputState) {onImageExpand={props.onImageExpand}/>));- const text = props.output ? (- <div className="max-h-80 overflow-auto text-muted-foreground">{props.output}</div>+}++function ToolOutput(props: ToolOutputState) {+ return props.output ? (+ <StructuredValue value={props.output} highlightJson />) : props.pending ? (<div className="text-muted-foreground italic">Loading output…</div>) : props.error ? (<div className="text-destructive">Couldn't load output: {props.error}</div>- ) : props.empty && images.length === 0 ? (+ ) : props.empty && props.images.length === 0 ? (<div className="text-muted-foreground italic">No output.</div>) : null;- return (- <>- {images}- {text}- </>- );}export function FetchedToolOutput(props: {@@ -187,6 +175,7 @@ export function FetchedToolOutput(props: {const { output } = useFetchedTurnItem(props.projectedItem, props.environmentId, props.source);return (<div className={cn("mt-1.5 space-y-1.5", monoClassName)}>+ <ToolOutputImages {...output} onImageExpand={props.onImageExpand} /><ToolOutput {...output} onImageExpand={props.onImageExpand} /></div>);@@ -205,14 +194,19 @@ function ToolCallBody(const call = toolCallLines({ command: props.command, args: props.args });return (<div className={cn("space-y-1.5", monoClassName)}>+ <ToolOutputImages {...props} />{call.command ? <ShellCommandBlock command={call.command} /> : null}{call.args ? (<div className="text-foreground/85">{call.args.map(([key, value]) => (<div key={key}><span className="text-muted-foreground">{key} </span>- {key === "code" && isJavaScriptReplTool(props.toolName) ? (+ {key === "html" ? (+ <StructuredValue value={value} language="html" />+ ) : key === "code" && isJavaScriptReplTool(props.toolName) ? (<StructuredValue value={value} language="typescript" />+ ) : isJsonToolText(value) ? (+ <StructuredValue value={value} language="json" />) : (value)}diff --git a/packages/client-runtime/src/work-log/itemDetail.test.ts b/packages/client-runtime/src/work-log/itemDetail.test.tsindex e1511170800..cfa00db4f8f 100644--- a/packages/client-runtime/src/work-log/itemDetail.test.ts+++ b/packages/client-runtime/src/work-log/itemDetail.test.ts@@ -2,7 +2,26 @@ import { describe, expect, it } from "vite-plus/test";import { ThreadId, TurnItemId, type OrchestrationV2TurnItem } from "@t3tools/contracts";import * as DateTime from "effect/DateTime";-import { turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";+import { isJsonToolText, turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";++describe("isJsonToolText", () => {+ it.each([+ ['{"screenshot":{"mimeType":"image/png"}}', true],+ ['[{"threadId":"thread-1"}]', true],+ ['{\n\n "status": "completed"\n}', true],+ ['{\n "count": 1\n}\n\n{\n "count": 2\n}', true],+ ["false", true],+ ["null", true],+ ["42", true],+ ['"completed"', true],+ ['{"partial":', false],+ ['{"count":1}\n\nFinished.', false],+ ["No output.", false],+ ["", false],+ ] as const)("classifies %j as JSON: %s", (text, expected) => {+ expect(isJsonToolText(text)).toBe(expected);+ });+});function toolItem(output: unknown): OrchestrationV2TurnItem {return {@@ -28,15 +47,54 @@ function toolItem(output: unknown): OrchestrationV2TurnItem {}describe("tool image output parsing", () => {- it.each(["array", "content"] as const)(+ it.each(["image-only", "summary", "json"] as const)(+ "keeps structured fields beside %s content",+ (shape) => {+ const content = [+ { type: "image", mimeType: "image/png" },+ ...(shape === "image-only"+ ? []+ : [{ type: "text", text: shape === "summary" ? "Sunny" : '{"temp":21}' }]),+ ];+ for (const structuredContent of [+ { temp: 21 },+ { type: "text", text: "Sunny", temp: 21 },+ { content: [{ type: "text", text: "Sunny" }], temp: 21 },+ ]) {+ const item = toolItem({ structuredContent, content });+ expect(turnItemOutputText(item)).toBe(JSON.stringify(structuredContent, null, 2));+ expect(turnItemOutputImages(item)).toHaveLength(1);+ }+ for (const structuredContent of [null, {}, [], ""]) {+ const item = toolItem({ structuredContent, content: [{ type: "text", text: "Sunny" }] });+ expect(turnItemOutputText(item)).toBe("Sunny");+ }+ expect(+ turnItemOutputText(+ toolItem({+ content: [{ type: "text", text: "File contents" }],+ structuredContent: { path: "read.ts", ready: true },+ }),+ ),+ ).toBe("File contents");+ },+ );+ it.each(["array", "content", "structured"] as const)("keeps markers for images beyond the served positions in %s output",(shape) => {// Detail responses retain image metadata after the server strips image bytes.const image = { type: "image", mimeType: "image/png" };const blocks = Array.from({ length: 8 }, () => ({ ...image }));- const wrap = (content: unknown[]) => (shape === "array" ? content : { content });+ const wrap = (content: unknown[]) =>+ shape === "array"+ ? content+ : shape === "structured"+ ? { content, structuredContent: { temp: 21 } }+ : { content };const withinLimit = toolItem(wrap(blocks));- expect(turnItemOutputText(withinLimit)).toBeNull();+ expect(turnItemOutputText(withinLimit)).toBe(+ shape === "structured" ? '{\n "temp": 21\n}' : null,+ );expect(turnItemOutputImages(withinLimit)).toHaveLength(8);const overflow = toolItem(@@ -48,7 +106,10 @@ describe("tool image output parsing", () => {]),);expect(turnItemOutputImages(overflow)).toHaveLength(8);- expect(turnItemOutputText(overflow)).toBe("Two additional screenshots:\n[image]\n[image]");+ expect(turnItemOutputText(overflow)).toBe(+ (shape === "structured" ? '{\n "temp": 21\n}' : "Two additional screenshots:") ++ "\n[image]\n[image]",+ );},);});diff --git a/packages/client-runtime/src/work-log/itemDetail.ts b/packages/client-runtime/src/work-log/itemDetail.tsindex 5e16cf538ca..16f79bd8d98 100644--- a/packages/client-runtime/src/work-log/itemDetail.ts+++ b/packages/client-runtime/src/work-log/itemDetail.ts@@ -39,6 +39,14 @@ function textFromBlocks(return textFromBlocks(value.content, depth + 1, imageBlocks);if (keys.length === 2 && keys.includes("content") && keys.includes("structuredContent")) {const text = textFromBlocks(value.content, depth + 1, imageBlocks);+ const images = Array.isArray(value.content)+ ? value.content.filter((block) => isRecord(block) && block.type === "image")+ : [];+ const structured = formatJsonValue(value.structuredContent);+ if (structured !== null && (images.length > 0 || !text?.trim())) {+ const markers = textFromBlocks(images, depth + 1, imageBlocks);+ return [structured, markers].filter(Boolean).join("\n");+ }return text?.trim() ? text : null;}return null;@@ -52,6 +60,12 @@ function parseJson(text: string): unknown {}}+export function isJsonToolText(text: string): boolean {+ if (parseJson(text) !== undefined) return true;+ const documents = text.trim().split(/\n\s*\n/);+ return documents.length > 1 && documents.every((document) => parseJson(document) !== undefined);+}+function prettyJsonText(text: string): string {const trimmed = text.trim();if (!/^[[{]/.test(trimmed)) return text;@@ -63,11 +77,9 @@ function prettyJsonText(text: string): string {return documents.map((document) => JSON.stringify(document, null, 2)).join("\n\n");}-function formatToolValue(value: unknown, hasImagePreview = false): string | null {+function formatJsonValue(value: unknown): string | null {if (value === undefined || value === null) return null;- if (hasImagePreview && isRecord(value) && value.type === "image") return null;- const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));- if (text !== null) return text.trim() ? prettyJsonText(text) : null;+ if (typeof value === "string" && !value.trim()) return null;let json: string | undefined;try {json = JSON.stringify(value, null, 2);@@ -78,6 +90,14 @@ function formatToolValue(value: unknown, hasImagePreview = false): string | nullreturn json;}+function formatToolValue(value: unknown, hasImagePreview = false): string | null {+ if (value === undefined || value === null) return null;+ if (hasImagePreview && isRecord(value) && value.type === "image") return null;+ const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));+ if (text !== null) return text.trim() ? prettyJsonText(text) : null;+ return formatJsonValue(value);+}+export function toolCallLines(input: {readonly command?: string | undefined;readonly args?: unknown;diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.tsindex 622bc63c933..d08ecd8ce87 100644--- a/packages/provider-acp/src/server/adapter.ts+++ b/packages/provider-acp/src/server/adapter.ts@@ -39,7 +39,11 @@ import { HTML_RENDER_TOOL_NAME } from "@t3tools/shared/htmlRender";import { modelSelectionsEqual } from "@t3tools/shared/model";import { type SelfInvocation, selfInvocationArgs } from "@t3tools/shared/nodeRuntime";import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation";-import { compactDynamicToolOutput, htmlRenderFromToolItem } from "@t3tools/shared/toolOutput";+import {+ compactDynamicToolOutput,+ htmlRenderFromToolItem,+ mcpToolResultOutput,+} from "@t3tools/shared/toolOutput";import { FILE_HEADERS_ONLY, formatPatch, structuredPatch } from "diff";import * as Cause from "effect/Cause";import * as Clock from "effect/Clock";@@ -923,8 +927,7 @@ function acpMcpToolCallOutput(rawOutput: unknown): unknown {if (record === undefined) return rawOutput;if (!("result" in record) && !("error" in record)) return rawOutput;const result = unknownRecord(record.result);- const resultOutput =- result === undefined ? undefined : (result.structuredContent ?? result.content ?? undefined);+ const resultOutput = result === undefined ? undefined : mcpToolResultOutput(result);const errorMessage = unknownRecord(record.error)?.message;if (typeof errorMessage !== "string") {return resultOutput ?? rawOutput;diff --git a/packages/provider-cursor/src/server/adapter.test.ts b/packages/provider-cursor/src/server/adapter.test.tsindex 4c421a352a8..69fa9cd63a8 100644--- a/packages/provider-cursor/src/server/adapter.test.ts+++ b/packages/provider-cursor/src/server/adapter.test.ts@@ -1,6 +1,7 @@import * as NodeServices from "@effect/platform-node/NodeServices";import { assert, describe, it } from "@effect/vitest";import type { InteractionUpdate } from "@cursor/sdk";+import { toolOutputImages } from "@t3tools/shared/toolOutput";import {EnvironmentId,MessageId,@@ -844,7 +845,18 @@ describe("CursorAdapterV2", () => {args: { city: "Berlin" },},...(type === "tool-call-completed"- ? { result: { status: "success" as const, value: { content: [], isError: false } } }+ ? {+ result: {+ status: "success" as const,+ value: {+ content: [+ { text: { text: "Preview complete" } },+ { image: { data: "iVBORw0KGgo=", mimeType: "image/png" } },+ ],+ isError: false,+ },+ },+ }: {}),},})),@@ -1155,6 +1167,9 @@ describe("CursorAdapterV2", () => {});assert.deepEqual(item.input, { city: "Berlin" });}+ assert.deepEqual(toolOutputImages(mcpItems.at(-1)?.output), [+ { mimeType: "image/png", data: "iVBORw0KGgo=" },+ ]);const fileSearchItems = events.flatMap((event) =>event.type === "turn_item.updated" &&event.turnItem.type === "file_search" &&diff --git a/packages/provider-cursor/src/server/adapter.ts b/packages/provider-cursor/src/server/adapter.tsindex 73f7ac9d976..e5b77e1041d 100644--- a/packages/provider-cursor/src/server/adapter.ts+++ b/packages/provider-cursor/src/server/adapter.ts@@ -377,6 +377,17 @@ function cursorToolOutput(toolCall: ToolCall): unknown {if (result === undefined) {return undefined;}+ if (toolCall.type === "mcp" && toolCall.result?.status === "success") {+ return {+ ...toolCall.result.value,+ content: toolCall.result.value.content.flatMap((part) => [+ ...(part.text === undefined ? [] : [{ type: "text", text: part.text.text }]),+ ...(part.image === undefined+ ? []+ : [{ type: "image", mimeType: part.image.mimeType, data: part.image.data }]),+ ]),+ };+ }return result.status === "success" ? result.value : result.error;}diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.tsindex 49cc166f805..6fb5263b6e3 100644--- a/packages/provider-opencode/src/server/adapter.test.ts+++ b/packages/provider-opencode/src/server/adapter.test.ts@@ -6,6 +6,7 @@ import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts";import { assert, describe, it } from "@effect/vitest";import * as NodeServices from "@effect/platform-node/NodeServices";import type { OpencodeClient, ToolPart } from "@opencode-ai/sdk/v2";+import { toolOutputImages } from "@t3tools/shared/toolOutput";import {CheckpointId,NodeId,@@ -1273,6 +1274,13 @@ describe("OpenCodeAdapterV2", () => {title: "Edit remote document",metadata: {},output,+ attachments: [+ {+ type: "file",+ mime: "image/png",+ url: "data:image/png;base64,iVBORw0KGgo=",+ },+ ],error: "failed",time: { start: 1, end: 2 },},@@ -1365,7 +1373,9 @@ describe("OpenCodeAdapterV2", () => {});assert.deepEqual(item.input, { document: "remote-doc" });}- assert.equal(items[1]?.output, output);+ assert.deepEqual(toolOutputImages(items[1]?.output), [+ { mimeType: "image/png", data: "iVBORw0KGgo=" },+ ]);assert.equal(items[3]?.toolName, "ambiguous_server_edit_document");assert.isUndefined(items[3]?.toolSource);assert.equal(statusReads, 1);diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.tsindex d0c646fa051..c4386e84854 100644--- a/packages/provider-opencode/src/server/adapter.ts+++ b/packages/provider-opencode/src/server/adapter.ts@@ -80,7 +80,7 @@ import {makeSubagentChildThread,subagentThreadTitle,} from "@t3tools/provider-core/server/subagentProjection";-import { openCodeCommandFailed, openCodeToolTurnItem } from "./toolItems.ts";+import { openCodeCommandFailed, openCodeToolOutput, openCodeToolTurnItem } from "./toolItems.ts";export const OPENCODE_PROVIDER = ProviderDriverKind.make("opencode");export const OPENCODE_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(OPENCODE_PROVIDER);@@ -1605,6 +1605,10 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(functiontoolName: part.tool.slice(serverName.replace(/[^a-zA-Z0-9_-]/g, "_").length + 1),title: toolTitle(part) === part.tool ? undefined : toolTitle(part),});+ const files =+ part.state.status === "completed"+ ? part.state.attachments?.map((file) => ({ uri: file.url, mime: file.mime }))+ : undefined;const turnItem: OrchestrationV2TurnItem = matchingServers?.length? {...base,@@ -1612,12 +1616,15 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function...presentation,toolName: part.tool,input,- ...(output === undefined ? {} : { output }),+ ...(output === undefined && !files?.length+ ? {}+ : { output: openCodeToolOutput(output, files) }),}: openCodeToolTurnItem(base, {name: part.tool,input,output,+ files,completedMetadata:part.state.status === "completed" ? part.state.metadata : undefined,});diff --git a/packages/provider-opencode/src/server/toolItems.test.ts b/packages/provider-opencode/src/server/toolItems.test.tsnew file mode 100644index 00000000000..8e043879b08--- /dev/null+++ b/packages/provider-opencode/src/server/toolItems.test.ts@@ -0,0 +1,43 @@+import { describe, expect, it } from "vite-plus/test";+import {+ MAX_TOOL_OUTPUT_IMAGES,+ MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH,+ toolOutputImages,+} from "@t3tools/shared/toolOutput";++import { openCodeToolOutput } from "./toolItems.ts";++describe("openCodeToolOutput", () => {+ it("preserves inline raster file content and leaves other files unserved", () => {+ const output = openCodeToolOutput("Preview complete", [+ { uri: "https://example.com/preview.png", mime: "image/png" },+ { uri: "file:///tmp/preview.png", mime: "image/png" },+ { uri: "data:image/svg+xml;base64,AAAA", mime: "image/svg+xml" },+ { uri: "data:image/png;base64,AAAA", mime: "image/jpeg" },+ { uri: "data:image/png;base64,", mime: "image/png" },+ { uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" },+ ]);+ expect(toolOutputImages(output)).toEqual([{ mimeType: "image/png", data: "iVBORw0KGgo=" }]);+ expect(output).toEqual({+ content: [+ { type: "text", text: "Preview complete" },+ { type: "image", mimeType: "image/png", data: "iVBORw0KGgo=" },+ ],+ });+ expect(openCodeToolOutput("Text result")).toBe("Text result");+ const oversized = "A".repeat(MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4);+ const bounded = openCodeToolOutput(+ "Preview complete",+ Array.from({ length: MAX_TOOL_OUTPUT_IMAGES + 2 }, (_, index) => ({+ uri: `data:image/png;base64,${index === 0 ? oversized : "AQ=="}`,+ mime: "image/png",+ })),+ );+ expect(typeof bounded === "object" ? bounded.content.length : 0).toBe(+ MAX_TOOL_OUTPUT_IMAGES + 1,+ );+ const images = toolOutputImages(bounded);+ expect(images[0]?.data?.length).toBe(oversized.length);+ expect(images[1]?.data).toBe("AQ==");+ });+});diff --git a/packages/provider-opencode/src/server/toolItems.ts b/packages/provider-opencode/src/server/toolItems.tsindex c4188914e5b..f651d6465b7 100644--- a/packages/provider-opencode/src/server/toolItems.ts+++ b/packages/provider-opencode/src/server/toolItems.ts@@ -1,6 +1,29 @@import type { OrchestrationV2TurnItem } from "@t3tools/contracts";import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation";import { formatReadToolLabel, formatSearchToolLabel } from "@t3tools/shared/toolActivity";+import {+ MAX_TOOL_OUTPUT_IMAGES,+ readToolOutputImage,+ type ToolOutputImage,+} from "@t3tools/shared/toolOutput";++export function openCodeToolOutput(+ output: string | undefined,+ files: ReadonlyArray<{ readonly uri: string; readonly mime: string }> = [],+) {+ const images: Array<ToolOutputImage & { type: "image" }> = [];+ for (const file of files) {+ const match = /^data:(image\/[a-z0-9.+-]+);base64,([A-Za-z0-9+/]+={0,2})$/i.exec(file.uri);+ if (match?.[1]?.toLowerCase() !== file.mime.toLowerCase()) continue;+ const image = readToolOutputImage({ type: "image", mimeType: file.mime, data: match?.[2] });+ if (image === null) continue;+ images.push({ type: "image", ...image });+ if (images.length === MAX_TOOL_OUTPUT_IMAGES) break;+ }+ return images.length === 0+ ? output+ : { content: [...(output ? [{ type: "text", text: output }] : []), ...images] };+}// Search results stay on the timeline wire, so keep their text a preview.const SEARCH_PREVIEW_MAX_CHARS = 8_000;@@ -13,7 +36,7 @@ type ToolItemBase = Omit<function openCodeToolProjectionKind(toolName: string,): "command_execution" | "file_change" | "file_search" | "web_search" | "dynamic_tool" {- if (resolveT3McpToolId(toolName) !== null) return "dynamic_tool";+ if (toolName.startsWith("mcp__") || resolveT3McpToolId(toolName) !== null) return "dynamic_tool";const normalized = toolName.toLowerCase();if (normalized === "todowrite") {return "dynamic_tool";@@ -98,6 +121,7 @@ export function openCodeToolTurnItem(readonly input: Record<string, unknown>;readonly output: string | undefined;readonly completedMetadata: unknown;+ readonly files?: ReadonlyArray<{ readonly uri: string; readonly mime: string }> | undefined;},): OrchestrationV2TurnItem {const { input, output } = tool;@@ -178,7 +202,9 @@ export function openCodeToolTurnItem(type: "dynamic_tool",toolName: tool.name,input,- ...(cleanedOutput === undefined ? {} : { output: cleanedOutput }),+ ...(cleanedOutput === undefined && !tool.files?.length+ ? {}+ : { output: openCodeToolOutput(cleanedOutput, tool.files) }),};}}diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.tsindex 81871b4b698..ddca16ed8fe 100644--- a/packages/provider-opencode/src/server/v2/adapter.ts+++ b/packages/provider-opencode/src/server/v2/adapter.ts@@ -996,7 +996,11 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Prturn: ActiveTurn,id: string,status: "running" | "completed" | "failed" | "interrupted",- result?: { readonly output: string | undefined; readonly metadata: unknown },+ result?: {+ readonly output: string | undefined;+ readonly metadata: unknown;+ readonly files?: ReadonlyArray<{ readonly uri: string; readonly mime: string }>;+ },) {const tool = turn.tools.get(id);if (tool === undefined) return;@@ -1018,6 +1022,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Prname: tool.name,input: tool.input,output: result?.output,+ files: result?.files,completedMetadata: status === "completed" ? result?.metadata : undefined,},),@@ -2097,6 +2102,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Prconst output = textOf(event.data.content);yield* emitTool(state, turn, event.data.id, "completed", {output,+ files: event.data.content.flatMap((part) => (part.type === "file" ? [part] : [])),metadata: event.data.metadata,});turn.tools.delete(event.data.id);@@ -2555,6 +2561,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Prif (part.state.status === "completed") {yield* emitTool(state, turn, part.id, "completed", {output: textOf(part.state.content),+ files: part.state.content.flatMap((content) =>+ content.type === "file" ? [content] : [],+ ),metadata: part.state.metadata,});turn.tools.delete(part.id);diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.tsindex 58d38fd03ad..546169a35b3 100644--- a/packages/provider-pi/src/server/adapter.test.ts+++ b/packages/provider-pi/src/server/adapter.test.ts@@ -2328,7 +2328,10 @@ describe("PiAdapterV2", () => {if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")return yield* Effect.die("Expected a read image tool item");assert.deepEqual(event.turnItem.output, { content: imageContent, structuredContent });- assert.equal(turnItemOutputText(event.turnItem), note);+ assert.equal(+ turnItemOutputText(event.turnItem),+ JSON.stringify(structuredContent, null, 2),+ );}}).pipe(Effect.scoped, Effect.provide(layerTest)),);@@ -2385,7 +2388,7 @@ describe("PiAdapterV2", () => {assert.equal(event.turnItem.toolName, tool.name);assert.equal(event.turnItem.status, "completed");assert.deepEqual(event.turnItem.output, { content, structuredContent: metadata });- assert.equal(turnItemOutputText(event.turnItem), "Screenshot captured");+ assert.equal(turnItemOutputText(event.turnItem), JSON.stringify(metadata, null, 2));const image = toolOutputImages(event.turnItem.output)[0];assert.equal(image?.mimeType, "image/png");assert.deepEqual(Buffer.from(image!.data!, "base64"), pixels);diff --git a/packages/shared/src/toolOutput.test.ts b/packages/shared/src/toolOutput.test.tsindex e09e05b891f..453695e397d 100644--- a/packages/shared/src/toolOutput.test.ts+++ b/packages/shared/src/toolOutput.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it } from "vite-plus/test";import {compactDynamicToolOutput,MAX_TOOL_OUTPUT_IMAGES,+ mcpToolResultOutput,toolOutputImages,toolOutputIndicatesFailure,} from "./toolOutput.ts";@@ -187,6 +188,13 @@ describe("toolOutputImages", () => {]);expect(toolOutputImages([mcpImage])).toEqual([{ mimeType: "image/png", data: "AAAA" }]);expect(toolOutputImages(mcpImage)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);+ const structuredContent = { screenshot: { mimeType: "image/png", width: 728, height: 354 } };+ const content = [text, mcpImage];+ const output = mcpToolResultOutput({ content, structuredContent });+ expect(toolOutputImages(output)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);+ expect(output).toEqual({ content, structuredContent });+ expect(mcpToolResultOutput({ content: [text], structuredContent })).toBe(structuredContent);+ expect(mcpToolResultOutput({ content })).toBe(content);});it("skips types the server must not serve inline", () => {diff --git a/packages/shared/src/toolOutput.ts b/packages/shared/src/toolOutput.tsindex b3a88e59de1..4c1ea03344c 100644--- a/packages/shared/src/toolOutput.ts+++ b/packages/shared/src/toolOutput.ts@@ -274,6 +274,16 @@ export function toolOutputImages(value: unknown): ReadonlyArray<ToolOutputImage>return toolOutputImageBlocks(value).flatMap((block) => readToolOutputImage(block) ?? []);}+export function mcpToolResultOutput(result: {+ readonly content?: unknown;+ readonly structuredContent?: unknown;+}): unknown {+ if (result.structuredContent != null && toolOutputImages(result.content).length > 0) {+ return { content: result.content, structuredContent: result.structuredContent };+ }+ return result.structuredContent ?? result.content;+}+export function omitToolOutputImageData(value: unknown): unknown {const omit = (block: unknown) => {const image = readToolOutputImage(block);