Skip to content
TypeScript

fix(tools): show images from MCP and OpenCode outputs (#30389)

Yash Singhd51bc58f5e05
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e4c45728c23..dc12d15d178 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -272,12 +272,18 @@ jobs:
env:
TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }}
run: |
+ set -o pipefail
filters=(--filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web')
if [[ -n "$TEST_PACKAGES" ]]; then
filters=()
for package in $TEST_PACKAGES; do filters+=(--filter "$package"); done
fi
- vp run --parallel --concurrency-limit 4 "${filters[@]}" test
+ # The runner's trace writer can crash on surrogate pairs in test output.
+ vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c '
+ import sys
+ for line in sys.stdin:
+ sys.stdout.write(line.encode("ascii", "backslashreplace").decode("ascii"))
+ '
# The web suite is the largest outside the server. In the shared Test job it
# started last and ran alone for about 40s after everything else finished.
diff --git a/apps/mobile/src/features/threads/thread-work-log.tsx b/apps/mobile/src/features/threads/thread-work-log.tsx
index 0ff0271f44f..ae943c81614 100644
--- a/apps/mobile/src/features/threads/thread-work-log.tsx
+++ b/apps/mobile/src/features/threads/thread-work-log.tsx
@@ -69,6 +69,7 @@ import {
workEntryRowLabel,
} from "../../lib/threadActivity";
import {
+ isJsonToolText,
toolCallLines,
turnItemOutputImages,
turnItemOutputText,
@@ -874,14 +875,16 @@ function WorkspacePreparationRetryButton(props: {
);
}
-function ReplCodeText({
+function ToolCodeText({
code,
+ language,
theme,
}: {
readonly code: string;
+ readonly language: "typescript" | "html" | "json";
readonly theme: "light" | "dark";
}) {
- const lines = useMarkdownCodeHighlight({ code, language: "typescript", theme, enabled: true });
+ const lines = useMarkdownCodeHighlight({ code, language, theme, enabled: true });
let offset = 0;
const highlighted = lines?.flatMap((line, lineIndex) => {
const separator = lineIndex > 0 ? "\n" : "";
@@ -1255,20 +1258,32 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
) : call ? (
[
call.command,
- ...(call.args ?? []).map(([key, value]) =>
- key === "code" &&
- shownItem.type === "dynamic_tool" &&
- isJavaScriptReplTool(shownItem.toolName) ? (
+ ...(call.args ?? []).map(([key, value]) => {
+ const language =
+ key === "html"
+ ? "html"
+ : key === "code" &&
+ shownItem.type === "dynamic_tool" &&
+ isJavaScriptReplTool(shownItem.toolName)
+ ? "typescript"
+ : isJsonToolText(value)
+ ? "json"
+ : null;
+ return language ? (
<View key={key}>
<Text className="font-mono text-2xs leading-normal text-foreground-muted">
{key}
</Text>
- <ReplCodeText code={value} theme={props.themeAppearance} />
+ <ToolCodeText
+ code={value}
+ language={language}
+ theme={props.themeAppearance}
+ />
</View>
) : (
`${key} ${value}`
- ),
- ),
+ );
+ }),
call.argsText,
]
.filter((line) => Boolean(line))
@@ -1281,7 +1296,11 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
selectable
className="font-mono text-2xs leading-normal text-foreground"
>
- {line}
+ {isJsonToolText(line) ? (
+ <ToolCodeText code={line} language="json" theme={props.themeAppearance} />
+ ) : (
+ line
+ )}
</Text>
) : (
line
@@ -1289,7 +1308,11 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
)
) : fullDetail ? (
<Text selectable className="font-mono text-2xs leading-normal text-foreground-muted">
- {fullDetail}
+ {isJsonToolText(fullDetail) ? (
+ <ToolCodeText code={fullDetail} language="json" theme={props.themeAppearance} />
+ ) : (
+ fullDetail
+ )}
</Text>
) : null}
{fetchedOutput ? (
@@ -1300,7 +1323,15 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
(!call || call.command || call.args || call.argsText) && "mt-1.5",
)}
>
- {fetchedOutput}
+ {isJsonToolText(fetchedOutput) ? (
+ <ToolCodeText
+ code={fetchedOutput}
+ language="json"
+ theme={props.themeAppearance}
+ />
+ ) : (
+ fetchedOutput
+ )}
</Text>
) : null}
{streamsOutput ? null : exitLine}
diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
index ef721644f6a..3f771d6abee 100644
--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
@@ -26,6 +26,7 @@ import type {
} from "@anthropic-ai/claude-agent-sdk/sdk-tools";
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
import {
ChatAttachmentId,
type ChatAttachment,
@@ -4844,11 +4845,19 @@ describe("ClaudeAdapterV2 background wake turns", () => {
{
type: "tool_result",
tool_use_id: id,
- content: "Weather result",
+ content: [
+ { type: "text", text: "Weather result" },
+ {
+ type: "image",
+ source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
+ },
+ ],
is_error: isError,
},
],
},
+ tool_use_result:
+ title === undefined ? { screenshot: { mimeType: "image/png" } } : undefined,
}),
);
yield* Queue.offer(
@@ -4881,6 +4890,9 @@ describe("ClaudeAdapterV2 background wake turns", () => {
assert.deepEqual(item.input, { city: "Berlin" });
}
assert.equal(new Set(items.map((item) => item.id)).size, 1);
+ assert.deepEqual(toolOutputImages(items.at(-1)?.output), [
+ { mimeType: "image/png", data: "iVBORw0KGgo=" },
+ ]);
}).pipe(
Effect.scoped,
Effect.provide(
diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
index d2ae14545d9..98c920f5053 100644
--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
@@ -1,4 +1,5 @@
import * as DocumentText from "../../provider/DocumentText.ts";
+import { mcpToolResultOutput, toolOutputImages } from "@t3tools/shared/toolOutput";
import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts";
import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts";
import {
@@ -3084,7 +3085,14 @@ function claudeNativeToolOutputValue(output: ClaudeNativeToolOutput): unknown |
case "none":
return undefined;
case "content_block":
+ return output.value;
case "structured_tool_use_result":
+ if (toolOutputImages(output.value).length === 0) {
+ return mcpToolResultOutput({
+ content: output.fallbackValue,
+ structuredContent: output.value,
+ });
+ }
return output.value;
default:
return assertNever(output);
diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
index 69fc35aa39b..0f8e0ef1f33 100644
--- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
@@ -50,6 +50,7 @@ import {
import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model";
import { appendGeneratedImageEmbeds } from "@t3tools/shared/generatedImageMarkdown";
import { resolveSpawnCommand } from "@t3tools/shared/shell";
+import { mcpToolResultOutput } from "@t3tools/shared/toolOutput";
import type {
ChatAttachment,
OrchestrationV2AppThread,
@@ -490,9 +491,7 @@ function codexMcpToolOutput(
const resultOutput =
item.result === null || item.result === undefined
? undefined
- : item.result.structuredContent !== null && item.result.structuredContent !== undefined
- ? item.result.structuredContent
- : item.result.content;
+ : mcpToolResultOutput(item.result);
if (item.error === null || item.error === undefined) {
return resultOutput;
diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts
index f9819070fd8..e4e0851e82c 100644
--- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts
+++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts
@@ -1,4 +1,5 @@
import * as NodeURL from "node:url";
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it } from "@effect/vitest";
@@ -2715,48 +2716,67 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
}).pipe(Effect.scoped),
);
- it.effect("keeps thread-scoped T3 waits as dynamic tools through their lifecycle", () =>
- Effect.gen(function* () {
- const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };
- const name = "t3-code-web_delegate_task";
- const input = { task: "Review", mode: "wait" };
- const { runtime, thread } = yield* resumed([
- out("session.prompt", { sessionID: SESSION, text: "<any>" }),
- promptAccepted,
- event("session.tool.input.started", { ...tool, name }),
- event("session.tool.called", { ...tool, name, input, executed: true }),
- event("session.tool.success", {
- ...tool,
- content: [{ type: "text", text: "Reviewed" }],
- metadata: {},
- executed: true,
- }),
- event("session.execution.succeeded", { sessionID: SESSION }),
- ]);
- const events = yield* runtime.events.pipe(
- Stream.takeUntil((event) => event.type === "turn.terminal"),
- Stream.runCollect,
- Effect.forkScoped,
- );
- yield* runtime.startTurn(turnInput(thread));
- const items = (yield* Fiber.join(events)).flatMap((event) =>
- event.type === "turn_item.updated" &&
- event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)
- ? [event.turnItem]
- : [],
- );
- assert.deepEqual(
- items.map((item) => item.type),
- ["dynamic_tool", "dynamic_tool", "dynamic_tool"],
- );
- assert.deepEqual(
- items.map((item) => item.status),
- ["running", "running", "completed"],
- );
- const last = items.at(-1);
- assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);
- assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);
- }).pipe(Effect.scoped),
+ it.effect.each(["delegate_task", "html_preview", "mcp__acme__edit_document"])(
+ "keeps MCP %s as a dynamic tool through its lifecycle",
+ (toolName) =>
+ Effect.gen(function* () {
+ const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };
+ const name = toolName.startsWith("mcp__") ? toolName : `t3-code-web_${toolName}`;
+ const input =
+ toolName !== "delegate_task"
+ ? { html: "<p>Preview</p>" }
+ : { task: "Review", mode: "wait" };
+ const { runtime, thread } = yield* resumed([
+ out("session.prompt", { sessionID: SESSION, text: "<any>" }),
+ promptAccepted,
+ event("session.tool.input.started", { ...tool, name }),
+ event("session.tool.called", { ...tool, name, input, executed: true }),
+ event(
+ "session.tool.success",
+ {
+ ...tool,
+ content: [
+ { type: "text", text: "Reviewed" },
+ ...(toolName !== "delegate_task"
+ ? [{ type: "file", uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" }]
+ : []),
+ ],
+ metadata: {},
+ executed: true,
+ },
+ undefined,
+ 2,
+ ),
+ event("session.execution.succeeded", { sessionID: SESSION }),
+ ]);
+ const events = yield* runtime.events.pipe(
+ Stream.takeUntil((event) => event.type === "turn.terminal"),
+ Stream.runCollect,
+ Effect.forkScoped,
+ );
+ yield* runtime.startTurn(turnInput(thread));
+ const items = (yield* Fiber.join(events)).flatMap((event) =>
+ event.type === "turn_item.updated" &&
+ event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)
+ ? [event.turnItem]
+ : [],
+ );
+ assert.deepEqual(
+ items.map((item) => item.type),
+ ["dynamic_tool", "dynamic_tool", "dynamic_tool"],
+ );
+ assert.deepEqual(
+ items.map((item) => item.status),
+ ["running", "running", "completed"],
+ );
+ const last = items.at(-1);
+ assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);
+ assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);
+ assert.deepEqual(
+ toolOutputImages(last?.type === "dynamic_tool" ? last.output : null),
+ toolName !== "delegate_task" ? [{ mimeType: "image/png", data: "iVBORw0KGgo=" }] : [],
+ );
+ }).pipe(Effect.scoped),
);
it.effect("stores a shell command that exits non-zero as failed", () =>
diff --git a/apps/web/src/components/chat/V2ItemInspector.tsx b/apps/web/src/components/chat/V2ItemInspector.tsx
index ee6072b926b..8d314a1bd1b 100644
--- a/apps/web/src/components/chat/V2ItemInspector.tsx
+++ b/apps/web/src/components/chat/V2ItemInspector.tsx
@@ -1,6 +1,7 @@
import type { ToolCallDiffTarget } from "../../diffPanelStore";
import type { EnvironmentId, OrchestrationV2ProjectedTurnItem } from "@t3tools/contracts";
import {
+ isJsonToolText,
toolCallLines,
turnItemDetailRevision,
turnItemNeedsDetailFetch,
@@ -43,7 +44,7 @@ function SyntaxTokens({
language,
}: {
readonly text: string;
- readonly language: "json" | "typescript";
+ readonly language: "json" | "typescript" | "html";
}) {
const resolvedTheme = useRenderedAppearance();
const highlighter = use(getSyntaxHighlighterPromise(language));
@@ -64,16 +65,6 @@ function SyntaxTokens({
const monoClassName =
"font-mono text-(length:--font-size-code,var(--text-2xs)) leading-relaxed whitespace-pre-wrap break-words select-text";
-function isJsonText(text: string): boolean {
- if (!text) return false;
- try {
- JSON.parse(text);
- return true;
- } catch {
- return false;
- }
-}
-
function StructuredValue({
value,
highlightJson = false,
@@ -81,10 +72,10 @@ function StructuredValue({
}: {
readonly value: unknown;
readonly highlightJson?: boolean;
- readonly language?: "typescript";
+ readonly language?: "typescript" | "html" | "json";
}) {
const text = typeof value === "string" ? value : JSON.stringify(value, null, 2);
- const isJson = highlightJson && isJsonText(text);
+ const isJson = highlightJson && isJsonToolText(text);
if (!text) return null;
return (
<pre className={cn("max-h-80 overflow-auto text-muted-foreground", monoClassName)}>
@@ -150,8 +141,8 @@ interface ToolOutputState {
readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
}
-function ToolOutput(props: ToolOutputState) {
- const images = props.images.map((resource) => (
+function ToolOutputImages(props: ToolOutputState) {
+ return props.images.map((resource) => (
<ChatMarkdownAssetImage
key={resource.index}
environmentId={props.environmentId}
@@ -161,21 +152,18 @@ function ToolOutput(props: ToolOutputState) {
onImageExpand={props.onImageExpand}
/>
));
- const text = props.output ? (
- <div className="max-h-80 overflow-auto text-muted-foreground">{props.output}</div>
+}
+
+function ToolOutput(props: ToolOutputState) {
+ return props.output ? (
+ <StructuredValue value={props.output} highlightJson />
) : props.pending ? (
<div className="text-muted-foreground italic">Loading output…</div>
) : props.error ? (
<div className="text-destructive">Couldn&apos;t load output: {props.error}</div>
- ) : props.empty && images.length === 0 ? (
+ ) : props.empty && props.images.length === 0 ? (
<div className="text-muted-foreground italic">No output.</div>
) : null;
- return (
- <>
- {images}
- {text}
- </>
- );
}
export function FetchedToolOutput(props: {
@@ -187,6 +175,7 @@ export function FetchedToolOutput(props: {
const { output } = useFetchedTurnItem(props.projectedItem, props.environmentId, props.source);
return (
<div className={cn("mt-1.5 space-y-1.5", monoClassName)}>
+ <ToolOutputImages {...output} onImageExpand={props.onImageExpand} />
<ToolOutput {...output} onImageExpand={props.onImageExpand} />
</div>
);
@@ -205,14 +194,19 @@ function ToolCallBody(
const call = toolCallLines({ command: props.command, args: props.args });
return (
<div className={cn("space-y-1.5", monoClassName)}>
+ <ToolOutputImages {...props} />
{call.command ? <ShellCommandBlock command={call.command} /> : null}
{call.args ? (
<div className="text-foreground/85">
{call.args.map(([key, value]) => (
<div key={key}>
<span className="text-muted-foreground">{key} </span>
- {key === "code" && isJavaScriptReplTool(props.toolName) ? (
+ {key === "html" ? (
+ <StructuredValue value={value} language="html" />
+ ) : key === "code" && isJavaScriptReplTool(props.toolName) ? (
<StructuredValue value={value} language="typescript" />
+ ) : isJsonToolText(value) ? (
+ <StructuredValue value={value} language="json" />
) : (
value
)}
diff --git a/packages/client-runtime/src/work-log/itemDetail.test.ts b/packages/client-runtime/src/work-log/itemDetail.test.ts
index e1511170800..cfa00db4f8f 100644
--- a/packages/client-runtime/src/work-log/itemDetail.test.ts
+++ b/packages/client-runtime/src/work-log/itemDetail.test.ts
@@ -2,7 +2,26 @@ import { describe, expect, it } from "vite-plus/test";
import { ThreadId, TurnItemId, type OrchestrationV2TurnItem } from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
-import { turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
+import { isJsonToolText, turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
+
+describe("isJsonToolText", () => {
+ it.each([
+ ['{"screenshot":{"mimeType":"image/png"}}', true],
+ ['[{"threadId":"thread-1"}]', true],
+ ['{\n\n "status": "completed"\n}', true],
+ ['{\n "count": 1\n}\n\n{\n "count": 2\n}', true],
+ ["false", true],
+ ["null", true],
+ ["42", true],
+ ['"completed"', true],
+ ['{"partial":', false],
+ ['{"count":1}\n\nFinished.', false],
+ ["No output.", false],
+ ["", false],
+ ] as const)("classifies %j as JSON: %s", (text, expected) => {
+ expect(isJsonToolText(text)).toBe(expected);
+ });
+});
function toolItem(output: unknown): OrchestrationV2TurnItem {
return {
@@ -28,15 +47,54 @@ function toolItem(output: unknown): OrchestrationV2TurnItem {
}
describe("tool image output parsing", () => {
- it.each(["array", "content"] as const)(
+ it.each(["image-only", "summary", "json"] as const)(
+ "keeps structured fields beside %s content",
+ (shape) => {
+ const content = [
+ { type: "image", mimeType: "image/png" },
+ ...(shape === "image-only"
+ ? []
+ : [{ type: "text", text: shape === "summary" ? "Sunny" : '{"temp":21}' }]),
+ ];
+ for (const structuredContent of [
+ { temp: 21 },
+ { type: "text", text: "Sunny", temp: 21 },
+ { content: [{ type: "text", text: "Sunny" }], temp: 21 },
+ ]) {
+ const item = toolItem({ structuredContent, content });
+ expect(turnItemOutputText(item)).toBe(JSON.stringify(structuredContent, null, 2));
+ expect(turnItemOutputImages(item)).toHaveLength(1);
+ }
+ for (const structuredContent of [null, {}, [], ""]) {
+ const item = toolItem({ structuredContent, content: [{ type: "text", text: "Sunny" }] });
+ expect(turnItemOutputText(item)).toBe("Sunny");
+ }
+ expect(
+ turnItemOutputText(
+ toolItem({
+ content: [{ type: "text", text: "File contents" }],
+ structuredContent: { path: "read.ts", ready: true },
+ }),
+ ),
+ ).toBe("File contents");
+ },
+ );
+ it.each(["array", "content", "structured"] as const)(
"keeps markers for images beyond the served positions in %s output",
(shape) => {
// Detail responses retain image metadata after the server strips image bytes.
const image = { type: "image", mimeType: "image/png" };
const blocks = Array.from({ length: 8 }, () => ({ ...image }));
- const wrap = (content: unknown[]) => (shape === "array" ? content : { content });
+ const wrap = (content: unknown[]) =>
+ shape === "array"
+ ? content
+ : shape === "structured"
+ ? { content, structuredContent: { temp: 21 } }
+ : { content };
const withinLimit = toolItem(wrap(blocks));
- expect(turnItemOutputText(withinLimit)).toBeNull();
+ expect(turnItemOutputText(withinLimit)).toBe(
+ shape === "structured" ? '{\n "temp": 21\n}' : null,
+ );
expect(turnItemOutputImages(withinLimit)).toHaveLength(8);
const overflow = toolItem(
@@ -48,7 +106,10 @@ describe("tool image output parsing", () => {
]),
);
expect(turnItemOutputImages(overflow)).toHaveLength(8);
- expect(turnItemOutputText(overflow)).toBe("Two additional screenshots:\n[image]\n[image]");
+ expect(turnItemOutputText(overflow)).toBe(
+ (shape === "structured" ? '{\n "temp": 21\n}' : "Two additional screenshots:") +
+ "\n[image]\n[image]",
+ );
},
);
});
diff --git a/packages/client-runtime/src/work-log/itemDetail.ts b/packages/client-runtime/src/work-log/itemDetail.ts
index 5e16cf538ca..16f79bd8d98 100644
--- a/packages/client-runtime/src/work-log/itemDetail.ts
+++ b/packages/client-runtime/src/work-log/itemDetail.ts
@@ -39,6 +39,14 @@ function textFromBlocks(
return textFromBlocks(value.content, depth + 1, imageBlocks);
if (keys.length === 2 && keys.includes("content") && keys.includes("structuredContent")) {
const text = textFromBlocks(value.content, depth + 1, imageBlocks);
+ const images = Array.isArray(value.content)
+ ? value.content.filter((block) => isRecord(block) && block.type === "image")
+ : [];
+ const structured = formatJsonValue(value.structuredContent);
+ if (structured !== null && (images.length > 0 || !text?.trim())) {
+ const markers = textFromBlocks(images, depth + 1, imageBlocks);
+ return [structured, markers].filter(Boolean).join("\n");
+ }
return text?.trim() ? text : null;
}
return null;
@@ -52,6 +60,12 @@ function parseJson(text: string): unknown {
}
}
+export function isJsonToolText(text: string): boolean {
+ if (parseJson(text) !== undefined) return true;
+ const documents = text.trim().split(/\n\s*\n/);
+ return documents.length > 1 && documents.every((document) => parseJson(document) !== undefined);
+}
+
function prettyJsonText(text: string): string {
const trimmed = text.trim();
if (!/^[[{]/.test(trimmed)) return text;
@@ -63,11 +77,9 @@ function prettyJsonText(text: string): string {
return documents.map((document) => JSON.stringify(document, null, 2)).join("\n\n");
}
-function formatToolValue(value: unknown, hasImagePreview = false): string | null {
+function formatJsonValue(value: unknown): string | null {
if (value === undefined || value === null) return null;
- if (hasImagePreview && isRecord(value) && value.type === "image") return null;
- const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));
- if (text !== null) return text.trim() ? prettyJsonText(text) : null;
+ if (typeof value === "string" && !value.trim()) return null;
let json: string | undefined;
try {
json = JSON.stringify(value, null, 2);
@@ -78,6 +90,14 @@ function formatToolValue(value: unknown, hasImagePreview = false): string | null
return json;
}
+function formatToolValue(value: unknown, hasImagePreview = false): string | null {
+ if (value === undefined || value === null) return null;
+ if (hasImagePreview && isRecord(value) && value.type === "image") return null;
+ const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));
+ if (text !== null) return text.trim() ? prettyJsonText(text) : null;
+ return formatJsonValue(value);
+}
+
export function toolCallLines(input: {
readonly command?: string | undefined;
readonly args?: unknown;
diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts
index 622bc63c933..d08ecd8ce87 100644
--- a/packages/provider-acp/src/server/adapter.ts
+++ b/packages/provider-acp/src/server/adapter.ts
@@ -39,7 +39,11 @@ import { HTML_RENDER_TOOL_NAME } from "@t3tools/shared/htmlRender";
import { modelSelectionsEqual } from "@t3tools/shared/model";
import { type SelfInvocation, selfInvocationArgs } from "@t3tools/shared/nodeRuntime";
import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation";
-import { compactDynamicToolOutput, htmlRenderFromToolItem } from "@t3tools/shared/toolOutput";
+import {
+ compactDynamicToolOutput,
+ htmlRenderFromToolItem,
+ mcpToolResultOutput,
+} from "@t3tools/shared/toolOutput";
import { FILE_HEADERS_ONLY, formatPatch, structuredPatch } from "diff";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
@@ -923,8 +927,7 @@ function acpMcpToolCallOutput(rawOutput: unknown): unknown {
if (record === undefined) return rawOutput;
if (!("result" in record) && !("error" in record)) return rawOutput;
const result = unknownRecord(record.result);
- const resultOutput =
- result === undefined ? undefined : (result.structuredContent ?? result.content ?? undefined);
+ const resultOutput = result === undefined ? undefined : mcpToolResultOutput(result);
const errorMessage = unknownRecord(record.error)?.message;
if (typeof errorMessage !== "string") {
return resultOutput ?? rawOutput;
diff --git a/packages/provider-cursor/src/server/adapter.test.ts b/packages/provider-cursor/src/server/adapter.test.ts
index 4c421a352a8..69fa9cd63a8 100644
--- a/packages/provider-cursor/src/server/adapter.test.ts
+++ b/packages/provider-cursor/src/server/adapter.test.ts
@@ -1,6 +1,7 @@
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, describe, it } from "@effect/vitest";
import type { InteractionUpdate } from "@cursor/sdk";
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
import {
EnvironmentId,
MessageId,
@@ -844,7 +845,18 @@ describe("CursorAdapterV2", () => {
args: { city: "Berlin" },
},
...(type === "tool-call-completed"
- ? { result: { status: "success" as const, value: { content: [], isError: false } } }
+ ? {
+ result: {
+ status: "success" as const,
+ value: {
+ content: [
+ { text: { text: "Preview complete" } },
+ { image: { data: "iVBORw0KGgo=", mimeType: "image/png" } },
+ ],
+ isError: false,
+ },
+ },
+ }
: {}),
},
})),
@@ -1155,6 +1167,9 @@ describe("CursorAdapterV2", () => {
});
assert.deepEqual(item.input, { city: "Berlin" });
}
+ assert.deepEqual(toolOutputImages(mcpItems.at(-1)?.output), [
+ { mimeType: "image/png", data: "iVBORw0KGgo=" },
+ ]);
const fileSearchItems = events.flatMap((event) =>
event.type === "turn_item.updated" &&
event.turnItem.type === "file_search" &&
diff --git a/packages/provider-cursor/src/server/adapter.ts b/packages/provider-cursor/src/server/adapter.ts
index 73f7ac9d976..e5b77e1041d 100644
--- a/packages/provider-cursor/src/server/adapter.ts
+++ b/packages/provider-cursor/src/server/adapter.ts
@@ -377,6 +377,17 @@ function cursorToolOutput(toolCall: ToolCall): unknown {
if (result === undefined) {
return undefined;
}
+ if (toolCall.type === "mcp" && toolCall.result?.status === "success") {
+ return {
+ ...toolCall.result.value,
+ content: toolCall.result.value.content.flatMap((part) => [
+ ...(part.text === undefined ? [] : [{ type: "text", text: part.text.text }]),
+ ...(part.image === undefined
+ ? []
+ : [{ type: "image", mimeType: part.image.mimeType, data: part.image.data }]),
+ ]),
+ };
+ }
return result.status === "success" ? result.value : result.error;
}
diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.ts
index 49cc166f805..6fb5263b6e3 100644
--- a/packages/provider-opencode/src/server/adapter.test.ts
+++ b/packages/provider-opencode/src/server/adapter.test.ts
@@ -6,6 +6,7 @@ import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts";
import { assert, describe, it } from "@effect/vitest";
import * as NodeServices from "@effect/platform-node/NodeServices";
import type { OpencodeClient, ToolPart } from "@opencode-ai/sdk/v2";
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
import {
CheckpointId,
NodeId,
@@ -1273,6 +1274,13 @@ describe("OpenCodeAdapterV2", () => {
title: "Edit remote document",
metadata: {},
output,
+ attachments: [
+ {
+ type: "file",
+ mime: "image/png",
+ url: "data:image/png;base64,iVBORw0KGgo=",
+ },
+ ],
error: "failed",
time: { start: 1, end: 2 },
},
@@ -1365,7 +1373,9 @@ describe("OpenCodeAdapterV2", () => {
});
assert.deepEqual(item.input, { document: "remote-doc" });
}
- assert.equal(items[1]?.output, output);
+ assert.deepEqual(toolOutputImages(items[1]?.output), [
+ { mimeType: "image/png", data: "iVBORw0KGgo=" },
+ ]);
assert.equal(items[3]?.toolName, "ambiguous_server_edit_document");
assert.isUndefined(items[3]?.toolSource);
assert.equal(statusReads, 1);
diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts
index d0c646fa051..c4386e84854 100644
--- a/packages/provider-opencode/src/server/adapter.ts
+++ b/packages/provider-opencode/src/server/adapter.ts
@@ -80,7 +80,7 @@ import {
makeSubagentChildThread,
subagentThreadTitle,
} from "@t3tools/provider-core/server/subagentProjection";
-import { openCodeCommandFailed, openCodeToolTurnItem } from "./toolItems.ts";
+import { openCodeCommandFailed, openCodeToolOutput, openCodeToolTurnItem } from "./toolItems.ts";
export const OPENCODE_PROVIDER = ProviderDriverKind.make("opencode");
export const OPENCODE_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(OPENCODE_PROVIDER);
@@ -1605,6 +1605,10 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function
toolName: part.tool.slice(serverName.replace(/[^a-zA-Z0-9_-]/g, "_").length + 1),
title: toolTitle(part) === part.tool ? undefined : toolTitle(part),
});
+ const files =
+ part.state.status === "completed"
+ ? part.state.attachments?.map((file) => ({ uri: file.url, mime: file.mime }))
+ : undefined;
const turnItem: OrchestrationV2TurnItem = matchingServers?.length
? {
...base,
@@ -1612,12 +1616,15 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function
...presentation,
toolName: part.tool,
input,
- ...(output === undefined ? {} : { output }),
+ ...(output === undefined && !files?.length
+ ? {}
+ : { output: openCodeToolOutput(output, files) }),
}
: openCodeToolTurnItem(base, {
name: part.tool,
input,
output,
+ files,
completedMetadata:
part.state.status === "completed" ? part.state.metadata : undefined,
});
diff --git a/packages/provider-opencode/src/server/toolItems.test.ts b/packages/provider-opencode/src/server/toolItems.test.ts
new file mode 100644
index 00000000000..8e043879b08
--- /dev/null
+++ b/packages/provider-opencode/src/server/toolItems.test.ts
@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vite-plus/test";
+import {
+ MAX_TOOL_OUTPUT_IMAGES,
+ MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH,
+ toolOutputImages,
+} from "@t3tools/shared/toolOutput";
+
+import { openCodeToolOutput } from "./toolItems.ts";
+
+describe("openCodeToolOutput", () => {
+ it("preserves inline raster file content and leaves other files unserved", () => {
+ const output = openCodeToolOutput("Preview complete", [
+ { uri: "https://example.com/preview.png", mime: "image/png" },
+ { uri: "file:///tmp/preview.png", mime: "image/png" },
+ { uri: "data:image/svg+xml;base64,AAAA", mime: "image/svg+xml" },
+ { uri: "data:image/png;base64,AAAA", mime: "image/jpeg" },
+ { uri: "data:image/png;base64,", mime: "image/png" },
+ { uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" },
+ ]);
+ expect(toolOutputImages(output)).toEqual([{ mimeType: "image/png", data: "iVBORw0KGgo=" }]);
+ expect(output).toEqual({
+ content: [
+ { type: "text", text: "Preview complete" },
+ { type: "image", mimeType: "image/png", data: "iVBORw0KGgo=" },
+ ],
+ });
+ expect(openCodeToolOutput("Text result")).toBe("Text result");
+ const oversized = "A".repeat(MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4);
+ const bounded = openCodeToolOutput(
+ "Preview complete",
+ Array.from({ length: MAX_TOOL_OUTPUT_IMAGES + 2 }, (_, index) => ({
+ uri: `data:image/png;base64,${index === 0 ? oversized : "AQ=="}`,
+ mime: "image/png",
+ })),
+ );
+ expect(typeof bounded === "object" ? bounded.content.length : 0).toBe(
+ MAX_TOOL_OUTPUT_IMAGES + 1,
+ );
+ const images = toolOutputImages(bounded);
+ expect(images[0]?.data?.length).toBe(oversized.length);
+ expect(images[1]?.data).toBe("AQ==");
+ });
+});
diff --git a/packages/provider-opencode/src/server/toolItems.ts b/packages/provider-opencode/src/server/toolItems.ts
index c4188914e5b..f651d6465b7 100644
--- a/packages/provider-opencode/src/server/toolItems.ts
+++ b/packages/provider-opencode/src/server/toolItems.ts
@@ -1,6 +1,29 @@
import type { OrchestrationV2TurnItem } from "@t3tools/contracts";
import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation";
import { formatReadToolLabel, formatSearchToolLabel } from "@t3tools/shared/toolActivity";
+import {
+ MAX_TOOL_OUTPUT_IMAGES,
+ readToolOutputImage,
+ type ToolOutputImage,
+} from "@t3tools/shared/toolOutput";
+
+export function openCodeToolOutput(
+ output: string | undefined,
+ files: ReadonlyArray<{ readonly uri: string; readonly mime: string }> = [],
+) {
+ const images: Array<ToolOutputImage & { type: "image" }> = [];
+ for (const file of files) {
+ const match = /^data:(image\/[a-z0-9.+-]+);base64,([A-Za-z0-9+/]+={0,2})$/i.exec(file.uri);
+ if (match?.[1]?.toLowerCase() !== file.mime.toLowerCase()) continue;
+ const image = readToolOutputImage({ type: "image", mimeType: file.mime, data: match?.[2] });
+ if (image === null) continue;
+ images.push({ type: "image", ...image });
+ if (images.length === MAX_TOOL_OUTPUT_IMAGES) break;
+ }
+ return images.length === 0
+ ? output
+ : { content: [...(output ? [{ type: "text", text: output }] : []), ...images] };
+}
// Search results stay on the timeline wire, so keep their text a preview.
const SEARCH_PREVIEW_MAX_CHARS = 8_000;
@@ -13,7 +36,7 @@ type ToolItemBase = Omit<
function openCodeToolProjectionKind(
toolName: string,
): "command_execution" | "file_change" | "file_search" | "web_search" | "dynamic_tool" {
- if (resolveT3McpToolId(toolName) !== null) return "dynamic_tool";
+ if (toolName.startsWith("mcp__") || resolveT3McpToolId(toolName) !== null) return "dynamic_tool";
const normalized = toolName.toLowerCase();
if (normalized === "todowrite") {
return "dynamic_tool";
@@ -98,6 +121,7 @@ export function openCodeToolTurnItem(
readonly input: Record<string, unknown>;
readonly output: string | undefined;
readonly completedMetadata: unknown;
+ readonly files?: ReadonlyArray<{ readonly uri: string; readonly mime: string }> | undefined;
},
): OrchestrationV2TurnItem {
const { input, output } = tool;
@@ -178,7 +202,9 @@ export function openCodeToolTurnItem(
type: "dynamic_tool",
toolName: tool.name,
input,
- ...(cleanedOutput === undefined ? {} : { output: cleanedOutput }),
+ ...(cleanedOutput === undefined && !tool.files?.length
+ ? {}
+ : { output: openCodeToolOutput(cleanedOutput, tool.files) }),
};
}
}
diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts
index 81871b4b698..ddca16ed8fe 100644
--- a/packages/provider-opencode/src/server/v2/adapter.ts
+++ b/packages/provider-opencode/src/server/v2/adapter.ts
@@ -996,7 +996,11 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
turn: ActiveTurn,
id: string,
status: "running" | "completed" | "failed" | "interrupted",
- result?: { readonly output: string | undefined; readonly metadata: unknown },
+ result?: {
+ readonly output: string | undefined;
+ readonly metadata: unknown;
+ readonly files?: ReadonlyArray<{ readonly uri: string; readonly mime: string }>;
+ },
) {
const tool = turn.tools.get(id);
if (tool === undefined) return;
@@ -1018,6 +1022,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
name: tool.name,
input: tool.input,
output: result?.output,
+ files: result?.files,
completedMetadata: status === "completed" ? result?.metadata : undefined,
},
),
@@ -2097,6 +2102,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
const output = textOf(event.data.content);
yield* emitTool(state, turn, event.data.id, "completed", {
output,
+ files: event.data.content.flatMap((part) => (part.type === "file" ? [part] : [])),
metadata: event.data.metadata,
});
turn.tools.delete(event.data.id);
@@ -2555,6 +2561,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
if (part.state.status === "completed") {
yield* emitTool(state, turn, part.id, "completed", {
output: textOf(part.state.content),
+ files: part.state.content.flatMap((content) =>
+ content.type === "file" ? [content] : [],
+ ),
metadata: part.state.metadata,
});
turn.tools.delete(part.id);
diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts
index 58d38fd03ad..546169a35b3 100644
--- a/packages/provider-pi/src/server/adapter.test.ts
+++ b/packages/provider-pi/src/server/adapter.test.ts
@@ -2328,7 +2328,10 @@ describe("PiAdapterV2", () => {
if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")
return yield* Effect.die("Expected a read image tool item");
assert.deepEqual(event.turnItem.output, { content: imageContent, structuredContent });
- assert.equal(turnItemOutputText(event.turnItem), note);
+ assert.equal(
+ turnItemOutputText(event.turnItem),
+ JSON.stringify(structuredContent, null, 2),
+ );
}
}).pipe(Effect.scoped, Effect.provide(layerTest)),
);
@@ -2385,7 +2388,7 @@ describe("PiAdapterV2", () => {
assert.equal(event.turnItem.toolName, tool.name);
assert.equal(event.turnItem.status, "completed");
assert.deepEqual(event.turnItem.output, { content, structuredContent: metadata });
- assert.equal(turnItemOutputText(event.turnItem), "Screenshot captured");
+ assert.equal(turnItemOutputText(event.turnItem), JSON.stringify(metadata, null, 2));
const image = toolOutputImages(event.turnItem.output)[0];
assert.equal(image?.mimeType, "image/png");
assert.deepEqual(Buffer.from(image!.data!, "base64"), pixels);
diff --git a/packages/shared/src/toolOutput.test.ts b/packages/shared/src/toolOutput.test.ts
index e09e05b891f..453695e397d 100644
--- a/packages/shared/src/toolOutput.test.ts
+++ b/packages/shared/src/toolOutput.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vite-plus/test";
import {
compactDynamicToolOutput,
MAX_TOOL_OUTPUT_IMAGES,
+ mcpToolResultOutput,
toolOutputImages,
toolOutputIndicatesFailure,
} from "./toolOutput.ts";
@@ -187,6 +188,13 @@ describe("toolOutputImages", () => {
]);
expect(toolOutputImages([mcpImage])).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
expect(toolOutputImages(mcpImage)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
+ const structuredContent = { screenshot: { mimeType: "image/png", width: 728, height: 354 } };
+ const content = [text, mcpImage];
+ const output = mcpToolResultOutput({ content, structuredContent });
+ expect(toolOutputImages(output)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
+ expect(output).toEqual({ content, structuredContent });
+ expect(mcpToolResultOutput({ content: [text], structuredContent })).toBe(structuredContent);
+ expect(mcpToolResultOutput({ content })).toBe(content);
});
it("skips types the server must not serve inline", () => {
diff --git a/packages/shared/src/toolOutput.ts b/packages/shared/src/toolOutput.ts
index b3a88e59de1..4c1ea03344c 100644
--- a/packages/shared/src/toolOutput.ts
+++ b/packages/shared/src/toolOutput.ts
@@ -274,6 +274,16 @@ export function toolOutputImages(value: unknown): ReadonlyArray<ToolOutputImage>
return toolOutputImageBlocks(value).flatMap((block) => readToolOutputImage(block) ?? []);
}
+export function mcpToolResultOutput(result: {
+ readonly content?: unknown;
+ readonly structuredContent?: unknown;
+}): unknown {
+ if (result.structuredContent != null && toolOutputImages(result.content).length > 0) {
+ return { content: result.content, structuredContent: result.structuredContent };
+ }
+ return result.structuredContent ?? result.content;
+}
+
export function omitToolOutputImageData(value: unknown): unknown {
const omit = (block: unknown) => {
const image = readToolOutputImage(block);