Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance tool item functionality to support image display

konstantin-paulusc893dd6537e3
diff --git a/apps/web/src/agent-chat/items.tsx b/apps/web/src/agent-chat/items.tsx
index da8cdb84..1af7b756 100644
--- a/apps/web/src/agent-chat/items.tsx
+++ b/apps/web/src/agent-chat/items.tsx
@@ -65,7 +65,7 @@ export function ReasoningItem(props: { item: Of<"reasoning">; active?: boolean }
export function ToolItem(props: { item: Of<"tool"> }) {
const [open, setOpen] = createSignal(false);
- const expandable = () => !!(props.item.detail || props.item.output);
+ const expandable = () => !!(props.item.detail || props.item.output || props.item.images?.length);
return (
<div class="pr-0.5">
<button
@@ -104,6 +104,16 @@ export function ToolItem(props: { item: Of<"tool"> }) {
{props.item.output}
</pre>
</Show>
+ <For each={props.item.images}>
+ {(image) => (
+ <img
+ src={`data:${image.mediaType};base64,${image.data}`}
+ alt={`${props.item.title} result`}
+ class="max-h-60 max-w-full self-start rounded-md border border-border object-contain"
+ loading="lazy"
+ />
+ )}
+ </For>
</div>
</Show>
</div>
diff --git a/packages/agent-chat/src/host/claude.ts b/packages/agent-chat/src/host/claude.ts
index 589d271d..e0478c30 100644
--- a/packages/agent-chat/src/host/claude.ts
+++ b/packages/agent-chat/src/host/claude.ts
@@ -13,7 +13,7 @@ import { query } from "@anthropic-ai/claude-agent-sdk";
import { HARNESS_LABELS } from "../protocol";
import { compareVersions, parseVersion, resolveBinary, resolveClaudeExecutable, runOnce } from "./env";
-import { QuestionBox, newItemId, summarizeInput, toolTitle, truncateDetail } from "./harness";
+import { QuestionBox, collectResult, newItemId, summarizeInput, toolTitle, truncateDetail } from "./harness";
import type {
CanUseTool,
@@ -457,13 +457,13 @@ class ClaudeSession implements HarnessSession {
if (block.type !== "tool_result") continue;
const started = turn.tools.get(block.tool_use_id);
if (!started) continue;
- const output = typeof block.content === "string"
- ? block.content
- : (block.content ?? []).map((part) => (part.type === "text" ? part.text : `[${part.type}]`)).join("\n");
+ const result = typeof block.content === "string"
+ ? { ...(block.content ? { output: truncateDetail(block.content) } : {}) }
+ : collectResult(block.content ?? []);
const item: Extract<Item, { kind: "tool" }> = {
...started,
status: block.is_error ? "failed" : "done",
- ...(output ? { output: truncateDetail(output) } : {}),
+ ...result,
};
turn.tools.delete(block.tool_use_id);
turn.emit({ type: "item.completed", item });
diff --git a/packages/agent-chat/src/host/harness.ts b/packages/agent-chat/src/host/harness.ts
index 8ae15e15..17310706 100644
--- a/packages/agent-chat/src/host/harness.ts
+++ b/packages/agent-chat/src/host/harness.ts
@@ -15,6 +15,7 @@ import type {
PendingRequest,
Question,
RequestResponse,
+ ToolImage,
TurnStatus,
} from "../protocol";
import type { HostEnv } from "./env";
@@ -66,6 +67,44 @@ export function truncateDetail(text: string | undefined): string | undefined {
return text.slice(0, DETAIL_MAX) + "…";
}
+/**
+ * The largest image (as base64) kept per tool result, and how many. Images
+ * ride inline in the transcript, which is replayed whenever a chat opens,
+ * so a capture is fine but a wall of full-size renders is not.
+ */
+export const IMAGE_MAX = 512 * 1024;
+export const IMAGES_MAX = 8;
+
+/**
+ * Splits a tool result's parts into the text shown and the images kept:
+ * text parts join with newlines, images within the cap ride along, and
+ * anything else (or an image over the cap) leaves a `[type]` marker.
+ */
+export function collectResult(parts: Array<{ type: string; text?: string; source?: unknown }>): { output?: string; images?: ToolImage[] } {
+ const lines: string[] = [];
+ const images: ToolImage[] = [];
+ for (const part of parts) {
+ if (part.type === "text") {
+ lines.push(part.text ?? "");
+ continue;
+ }
+ const image = part.type === "image" ? base64Image(part.source) : null;
+ if (image && image.data.length <= IMAGE_MAX && images.length < IMAGES_MAX) {
+ images.push(image);
+ continue;
+ }
+ lines.push(`[${part.type}]`);
+ }
+ const output = truncateDetail(lines.join("\n").trim() || undefined);
+ return { ...(output ? { output } : {}), ...(images.length ? { images } : {}) };
+}
+
+function base64Image(source: unknown): ToolImage | null {
+ if (!source || typeof source !== "object") return null;
+ const { type, media_type: mediaType, data } = source as Record<string, unknown>;
+ return type === "base64" && typeof mediaType === "string" && typeof data === "string" ? { mediaType, data } : null;
+}
+
export function newItemId(prefix: string): string {
return `${prefix}-${randomUUID().slice(0, 12)}`;
}
diff --git a/packages/agent-chat/src/protocol.ts b/packages/agent-chat/src/protocol.ts
index 863982f5..460822fc 100644
--- a/packages/agent-chat/src/protocol.ts
+++ b/packages/agent-chat/src/protocol.ts
@@ -42,11 +42,13 @@ export type ChatSummary = {
export type ToolStatus = "running" | "done" | "failed";
+export type ToolImage = { mediaType: string; data: string };
+
export type Item =
| { id: string; kind: "user"; text: string; attachments?: string[] }
| { id: string; kind: "assistant"; text: string }
| { id: string; kind: "reasoning"; text: string }
- | { id: string; kind: "tool"; name: string; title: string; detail?: string; output?: string; status: ToolStatus }
+ | { id: string; kind: "tool"; name: string; title: string; detail?: string; output?: string; images?: ToolImage[]; status: ToolStatus }
| { id: string; kind: "question"; questions: Question[]; answers: Record<string, string[]> | null }
| { id: string; kind: "notice"; level: "info" | "error"; text: string };
diff --git a/packages/agent-chat/test/harness.test.ts b/packages/agent-chat/test/harness.test.ts
new file mode 100644
index 00000000..f68b2ef4
--- /dev/null
+++ b/packages/agent-chat/test/harness.test.ts
@@ -0,0 +1,43 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { describe, expect, it } from "vitest";
+
+import { DETAIL_MAX, IMAGES_MAX, IMAGE_MAX, collectResult } from "../src/host/harness";
+
+const png = (data = "iVBORw0KGgo=") => ({ type: "image", source: { type: "base64", media_type: "image/png", data } });
+
+describe("collectResult", () => {
+ it("joins text parts and keeps images that fit", () => {
+ const result = collectResult([{ type: "text", text: "read it" }, png()]);
+ expect(result).toEqual({ output: "read it", images: [{ mediaType: "image/png", data: "iVBORw0KGgo=" }] });
+ });
+
+ it("returns only images when there is no text", () => {
+ expect(collectResult([png()])).toEqual({ images: [{ mediaType: "image/png", data: "iVBORw0KGgo=" }] });
+ });
+
+ it("leaves a marker for an image over the cap, and for other part kinds", () => {
+ const result = collectResult([png("x".repeat(IMAGE_MAX + 1)), { type: "document", source: "not-an-image" }]);
+ expect(result).toEqual({ output: "[image]\n[document]" });
+ });
+
+ it("keeps at most IMAGES_MAX images", () => {
+ const parts = Array.from({ length: IMAGES_MAX + 2 }, () => png());
+ const result = collectResult(parts);
+ expect(result.images).toHaveLength(IMAGES_MAX);
+ expect(result.output).toBe("[image]\n[image]");
+ });
+
+ it("truncates long text like any other output", () => {
+ const result = collectResult([{ type: "text", text: "y".repeat(DETAIL_MAX + 10) }]);
+ expect(result.output).toHaveLength(DETAIL_MAX + 1);
+ expect(result.output?.endsWith("…")).toBe(true);
+ });
+
+ it("drops empty output", () => {
+ expect(collectResult([])).toEqual({});
+ expect(collectResult([{ type: "text", text: " " }])).toEqual({});
+ });
+});