The VS Code of video editing, built for agents.
TypeScript
Enhance tool item functionality to support image display
konstantin-paulusc893dd6537e3
diff --git a/apps/web/src/agent-chat/items.tsx b/apps/web/src/agent-chat/items.tsxindex da8cdb84..1af7b756 100644--- a/apps/web/src/agent-chat/items.tsx+++ b/apps/web/src/agent-chat/items.tsx@@ -65,7 +65,7 @@ export function ReasoningItem(props: { item: Of<"reasoning">; active?: boolean }export function ToolItem(props: { item: Of<"tool"> }) {const [open, setOpen] = createSignal(false);- const expandable = () => !!(props.item.detail || props.item.output);+ const expandable = () => !!(props.item.detail || props.item.output || props.item.images?.length);return (<div class="pr-0.5"><button@@ -104,6 +104,16 @@ export function ToolItem(props: { item: Of<"tool"> }) {{props.item.output}</pre></Show>+ <For each={props.item.images}>+ {(image) => (+ <img+ src={`data:${image.mediaType};base64,${image.data}`}+ alt={`${props.item.title} result`}+ class="max-h-60 max-w-full self-start rounded-md border border-border object-contain"+ loading="lazy"+ />+ )}+ </For></div></Show></div>diff --git a/packages/agent-chat/src/host/claude.ts b/packages/agent-chat/src/host/claude.tsindex 589d271d..e0478c30 100644--- a/packages/agent-chat/src/host/claude.ts+++ b/packages/agent-chat/src/host/claude.ts@@ -13,7 +13,7 @@ import { query } from "@anthropic-ai/claude-agent-sdk";import { HARNESS_LABELS } from "../protocol";import { compareVersions, parseVersion, resolveBinary, resolveClaudeExecutable, runOnce } from "./env";-import { QuestionBox, newItemId, summarizeInput, toolTitle, truncateDetail } from "./harness";+import { QuestionBox, collectResult, newItemId, summarizeInput, toolTitle, truncateDetail } from "./harness";import type {CanUseTool,@@ -457,13 +457,13 @@ class ClaudeSession implements HarnessSession {if (block.type !== "tool_result") continue;const started = turn.tools.get(block.tool_use_id);if (!started) continue;- const output = typeof block.content === "string"- ? block.content- : (block.content ?? []).map((part) => (part.type === "text" ? part.text : `[${part.type}]`)).join("\n");+ const result = typeof block.content === "string"+ ? { ...(block.content ? { output: truncateDetail(block.content) } : {}) }+ : collectResult(block.content ?? []);const item: Extract<Item, { kind: "tool" }> = {...started,status: block.is_error ? "failed" : "done",- ...(output ? { output: truncateDetail(output) } : {}),+ ...result,};turn.tools.delete(block.tool_use_id);turn.emit({ type: "item.completed", item });diff --git a/packages/agent-chat/src/host/harness.ts b/packages/agent-chat/src/host/harness.tsindex 8ae15e15..17310706 100644--- a/packages/agent-chat/src/host/harness.ts+++ b/packages/agent-chat/src/host/harness.ts@@ -15,6 +15,7 @@ import type {PendingRequest,Question,RequestResponse,+ ToolImage,TurnStatus,} from "../protocol";import type { HostEnv } from "./env";@@ -66,6 +67,44 @@ export function truncateDetail(text: string | undefined): string | undefined {return text.slice(0, DETAIL_MAX) + "…";}+/**+ * The largest image (as base64) kept per tool result, and how many. Images+ * ride inline in the transcript, which is replayed whenever a chat opens,+ * so a capture is fine but a wall of full-size renders is not.+ */+export const IMAGE_MAX = 512 * 1024;+export const IMAGES_MAX = 8;++/**+ * Splits a tool result's parts into the text shown and the images kept:+ * text parts join with newlines, images within the cap ride along, and+ * anything else (or an image over the cap) leaves a `[type]` marker.+ */+export function collectResult(parts: Array<{ type: string; text?: string; source?: unknown }>): { output?: string; images?: ToolImage[] } {+ const lines: string[] = [];+ const images: ToolImage[] = [];+ for (const part of parts) {+ if (part.type === "text") {+ lines.push(part.text ?? "");+ continue;+ }+ const image = part.type === "image" ? base64Image(part.source) : null;+ if (image && image.data.length <= IMAGE_MAX && images.length < IMAGES_MAX) {+ images.push(image);+ continue;+ }+ lines.push(`[${part.type}]`);+ }+ const output = truncateDetail(lines.join("\n").trim() || undefined);+ return { ...(output ? { output } : {}), ...(images.length ? { images } : {}) };+}++function base64Image(source: unknown): ToolImage | null {+ if (!source || typeof source !== "object") return null;+ const { type, media_type: mediaType, data } = source as Record<string, unknown>;+ return type === "base64" && typeof mediaType === "string" && typeof data === "string" ? { mediaType, data } : null;+}+export function newItemId(prefix: string): string {return `${prefix}-${randomUUID().slice(0, 12)}`;}diff --git a/packages/agent-chat/src/protocol.ts b/packages/agent-chat/src/protocol.tsindex 863982f5..460822fc 100644--- a/packages/agent-chat/src/protocol.ts+++ b/packages/agent-chat/src/protocol.ts@@ -42,11 +42,13 @@ export type ChatSummary = {export type ToolStatus = "running" | "done" | "failed";+export type ToolImage = { mediaType: string; data: string };+export type Item =| { id: string; kind: "user"; text: string; attachments?: string[] }| { id: string; kind: "assistant"; text: string }| { id: string; kind: "reasoning"; text: string }- | { id: string; kind: "tool"; name: string; title: string; detail?: string; output?: string; status: ToolStatus }+ | { id: string; kind: "tool"; name: string; title: string; detail?: string; output?: string; images?: ToolImage[]; status: ToolStatus }| { id: string; kind: "question"; questions: Question[]; answers: Record<string, string[]> | null }| { id: string; kind: "notice"; level: "info" | "error"; text: string };diff --git a/packages/agent-chat/test/harness.test.ts b/packages/agent-chat/test/harness.test.tsnew file mode 100644index 00000000..f68b2ef4--- /dev/null+++ b/packages/agent-chat/test/harness.test.ts@@ -0,0 +1,43 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { describe, expect, it } from "vitest";++import { DETAIL_MAX, IMAGES_MAX, IMAGE_MAX, collectResult } from "../src/host/harness";++const png = (data = "iVBORw0KGgo=") => ({ type: "image", source: { type: "base64", media_type: "image/png", data } });++describe("collectResult", () => {+ it("joins text parts and keeps images that fit", () => {+ const result = collectResult([{ type: "text", text: "read it" }, png()]);+ expect(result).toEqual({ output: "read it", images: [{ mediaType: "image/png", data: "iVBORw0KGgo=" }] });+ });++ it("returns only images when there is no text", () => {+ expect(collectResult([png()])).toEqual({ images: [{ mediaType: "image/png", data: "iVBORw0KGgo=" }] });+ });++ it("leaves a marker for an image over the cap, and for other part kinds", () => {+ const result = collectResult([png("x".repeat(IMAGE_MAX + 1)), { type: "document", source: "not-an-image" }]);+ expect(result).toEqual({ output: "[image]\n[document]" });+ });++ it("keeps at most IMAGES_MAX images", () => {+ const parts = Array.from({ length: IMAGES_MAX + 2 }, () => png());+ const result = collectResult(parts);+ expect(result.images).toHaveLength(IMAGES_MAX);+ expect(result.output).toBe("[image]\n[image]");+ });++ it("truncates long text like any other output", () => {+ const result = collectResult([{ type: "text", text: "y".repeat(DETAIL_MAX + 10) }]);+ expect(result.output).toHaveLength(DETAIL_MAX + 1);+ expect(result.output?.endsWith("…")).toBe(true);+ });++ it("drops empty output", () => {+ expect(collectResult([])).toEqual({});+ expect(collectResult([{ type: "text", text: " " }])).toEqual({});+ });+});