The VS Code of video editing, built for agents.
TypeScript
Add file attachment handling and paste support in agent chat
konstantin-paulus9bde5fe6ef5f
diff --git a/apps/web/src/agent-chat/attachments.tsx b/apps/web/src/agent-chat/attachments.tsxindex e0832379..6eb98152 100644--- a/apps/web/src/agent-chat/attachments.tsx+++ b/apps/web/src/agent-chat/attachments.tsx@@ -4,6 +4,7 @@import { Show, createSignal } from "solid-js";+import { toast } from "somoto";import { pickFiles } from "@diffusionstudio/assets";@@ -11,17 +12,21 @@ import { Icon } from "@/components/ui/icon";import { RemoveButton } from "@/components/ui/remove-button";import { cx } from "@/lib/cva";+import { client, hasHost } from "./connection";+/*** A file or folder dropped onto a composer. Only what the tile and the* handoff need: the name to label it, the kind and extension to draw it, and* the path to send — null off the desktop, where the browser will not say- * where a dropped file lives.+ * where a dropped file lives. An image picked, dropped or pasted also carries+ * an object URL for its thumbnail; one rebuilt from a path does not.*/export type Attachment = {key: string;name: string;kind: "file" | "folder";path: string | null;+ preview?: string;};/** The paths worth sending: attachments the shell could locate. */@@ -57,9 +62,66 @@ export function droppedAttachments(event: DragEvent): Attachment[] {}/** A file picked or dropped without the entry API, which only a drop has. */-function fileAttachment(file: File, kind: Attachment["kind"] = "file", name = file.name): Attachment {- const path = window.desktop?.getPathForFile(file) || null;- return { key: path ?? `${name}:${file.size}:${file.lastModified}`, name, kind, path };+function fileAttachment(file: File, kind: Attachment["kind"] = "file", name = file.name, path = window.desktop?.getPathForFile(file) || null): Attachment {+ const preview = kind === "file" && file.type.startsWith("image/") ? URL.createObjectURL(file) : undefined;+ return { key: path ?? `${name}:${file.size}:${file.lastModified}`, name, kind, path, ...(preview ? { preview } : {}) };+}++/**+ * The files on the clipboard. None when the paste is text: rich text from a+ * document app carries a picture of itself alongside, and that is not what+ * the user meant to paste.+ */+function pastedFiles(event: ClipboardEvent): File[] {+ const data = event.clipboardData;+ if (!data || data.types.includes("text/rtf")) return [];+ return Array.from(data.files);+}++/**+ * Pasted files as attachments. A file copied in the Finder has a path and is+ * sent as it is; a screenshot or an image copied from a page is only bytes,+ * so it is uploaded to the agent host, which may not share this disk, and+ * the path it was written to is what gets sent.+ */+async function pastedAttachments(files: File[]): Promise<Attachment[]> {+ return Promise.all(+ files.map(async (file) => {+ const path = window.desktop?.getPathForFile(file) || null;+ if (path) return file.type.startsWith("image/") ? fileAttachment(file, "file", file.name, path) : attachmentFromPath(path);+ if (!hasHost()) return fileAttachment(file);+ const saved = await client.request("attachments.upload", { name: file.name, data: await base64(file) });+ return fileAttachment(file, "file", file.name, saved.path);+ }),+ );+}++/** A file's bytes as base64, without the data URL's prefix. */+function base64(file: File): Promise<string> {+ return new Promise((resolve, reject) => {+ const reader = new FileReader();+ reader.onload = () => {+ const url = reader.result as string;+ resolve(url.slice(url.indexOf(",") + 1));+ };+ reader.onerror = () => reject(reader.error);+ reader.readAsDataURL(file);+ });+}++/**+ * A paste handler for a composer: files go to `onPaste`, text is left to the+ * textarea. A paste that fails to save is a toast, not a lost image.+ */+export function createPasteHandler(onPaste: (pasted: Attachment[]) => void) {+ return (event: ClipboardEvent) => {+ const files = pastedFiles(event);+ if (files.length === 0) return;+ event.preventDefault();+ pastedAttachments(files).then(onPaste, (error: Error) =>+ toast.error("Could not paste the file", { description: error.message }),+ );+ };}/** Files chosen from the system picker, as attachments; folders can only be dropped. */@@ -151,9 +213,9 @@ type AttachmentTileProps = {};/**- * One dropped file or folder: a grey square with a folder mark, or the file's- * type in the middle. There is no thumbnail to show — nothing is loaded — so- * the name is in the tooltip and the remove button appears on hover, as it+ * One attached file or folder: the image itself when there is one to show,+ * else a grey square with a folder mark, or the file's type in the middle.+ * The name is in the tooltip and the remove button appears on hover, as it* does on the generation composer's reference images.*/export function AttachmentTile(props: AttachmentTileProps) {@@ -164,14 +226,21 @@ export function AttachmentTile(props: AttachmentTileProps) {><div class="grid size-full place-items-center overflow-hidden rounded-lg bg-input text-muted-foreground"><Show- when={props.attachment.kind === "folder"}+ when={props.attachment.preview}fallback={- <span class="max-w-9 truncate px-0.5 text-[9px] font-450 uppercase tracking-wide">- {fileType(props.attachment.name)}- </span>+ <Show+ when={props.attachment.kind === "folder"}+ fallback={+ <span class="max-w-9 truncate px-0.5 text-[9px] font-450 uppercase tracking-wide">+ {fileType(props.attachment.name)}+ </span>+ }+ >+ <Icon name="navigation.folder" class="size-6" />+ </Show>}>- <Icon name="navigation.folder" class="size-6" />+ {(src) => <img src={src()} alt="" draggable={false} class="size-full object-cover" />}</Show></div><RemoveButtondiff --git a/apps/web/src/agent-chat/composer.tsx b/apps/web/src/agent-chat/composer.tsxindex 322cfda3..a3e05a41 100644--- a/apps/web/src/agent-chat/composer.tsx+++ b/apps/web/src/agent-chat/composer.tsx@@ -13,7 +13,7 @@ import { Icon } from "@/components/ui/icon";import type { ModelRef } from "@diffusionstudio/agent-chat";-import { AttachmentTile, DropOverlay, createDropZone, mergeAttachments, pickAttachments, type Attachment } from "./attachments";+import { AttachmentTile, DropOverlay, createDropZone, createPasteHandler, mergeAttachments, pickAttachments, type Attachment } from "./attachments";import { ModelPicker } from "./model-picker";const MAX_HEIGHT_PX = 160;@@ -40,6 +40,7 @@ export function Composer(props: ComposerProps) {const attach = (added: Attachment[]) => props.onAttachments(mergeAttachments(props.attachments, added));const drop = createDropZone(attach);+ const paste = createPasteHandler(attach);const canSend = () => !props.running && !props.blocked && props.model !== null && (props.text.trim().length > 0 || props.attachments.length > 0);@@ -73,6 +74,7 @@ export function Composer(props: ComposerProps) {onDragEnter={drop.onDragEnter}onDragLeave={drop.onDragLeave}onDrop={drop.onDrop}+ onPaste={paste}><Show when={props.attachments.length > 0}>{/* The remove buttons overhang the tiles' corners, and a scrollingdiff --git a/apps/web/src/agent-chat/index.ts b/apps/web/src/agent-chat/index.tsindex c2126d3a..fb33be2f 100644--- a/apps/web/src/agent-chat/index.ts+++ b/apps/web/src/agent-chat/index.ts@@ -26,6 +26,7 @@ export {DropOverlay,attachmentPaths,createDropZone,+ createPasteHandler,droppedAttachments,mergeAttachments,pickAttachments,diff --git a/apps/web/src/components/dashboard/home-view.tsx b/apps/web/src/components/dashboard/home-view.tsxindex dc6f55a0..310883a4 100644--- a/apps/web/src/components/dashboard/home-view.tsx+++ b/apps/web/src/components/dashboard/home-view.tsx@@ -34,6 +34,7 @@ import {ModelPicker,attachmentPaths,createDropZone,+ createPasteHandler,currentModel,mergeAttachments,pickAttachments,@@ -112,6 +113,7 @@ export function DashboardHomeView() {const attach = (added: Attachment[]) => setAttachments((current) => mergeAttachments(current, added));const drop = createDropZone(attach);+ const paste = createPasteHandler(attach);let textarea: HTMLTextAreaElement | undefined;@@ -359,6 +361,7 @@ export function DashboardHomeView() {onDragEnter={drop.onDragEnter}onDragLeave={drop.onDragLeave}onDrop={drop.onDrop}+ onPaste={paste}><Show when={attachments().length > 0}>{/* The remove buttons overhang the tiles' corners, and adiff --git a/packages/agent-chat/src/host/host.ts b/packages/agent-chat/src/host/host.tsindex e43264e1..4baa4af5 100644--- a/packages/agent-chat/src/host/host.ts+++ b/packages/agent-chat/src/host/host.ts@@ -9,6 +9,9 @@// and they can come and go at any time without disturbing a turn.import { randomUUID } from "node:crypto";+import { mkdir, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { basename, join } from "node:path";import { isPersistedEvent, reduce } from "../reduce";import { HARNESS_IDS, HARNESS_LABELS, isHarnessId, titleFor } from "../protocol";@@ -60,6 +63,8 @@ export type AgentHostOptions = {idleMs?: number;/** How old a probe may be before `refresh` re-runs it. */probeMaxAgeMs?: number;+ /** Where uploaded attachments are written; default a folder in the system temp dir. */+ uploadDir?: string;log?: (message: string) => void;};@@ -191,6 +196,8 @@ export class AgentHost {return this.send(connection, p as MethodParams<"turn.send">) as Promise<R>;case "turn.interrupt":return this.interrupt(requireString(p, "chatId")) as Promise<R>;+ case "attachments.upload":+ return this.upload(p as MethodParams<"attachments.upload">) as Promise<R>;case "request.respond": {const { chatId, requestId, response } = p as MethodParams<"request.respond">;return this.respond(requireString({ chatId }, "chatId"), requireString({ requestId }, "requestId"), response) as R;@@ -200,6 +207,23 @@ export class AgentHost {}}+ // ---------------------------------------------------------------------+ // Attachments++ /**+ * A folder of its own per upload keeps the file's name — `image.png` for a+ * screenshot — without two uploads landing on the same path.+ */+ private async upload(params: MethodParams<"attachments.upload">): Promise<{ path: string }> {+ const name = basename(requireString(params, "name")).replace(/^\.+/, "") || "pasted";+ const data = requireString(params, "data");+ const dir = join(this.options.uploadDir ?? join(tmpdir(), "diffusion-studio-pasted"), randomUUID());+ await mkdir(dir, { recursive: true });+ const path = join(dir, name);+ await writeFile(path, Buffer.from(data, "base64"));+ return { path };+ }+// ---------------------------------------------------------------------// Harnessesdiff --git a/packages/agent-chat/src/host/index.ts b/packages/agent-chat/src/host/index.tsindex 298d1b7c..6b5756c9 100644--- a/packages/agent-chat/src/host/index.ts+++ b/packages/agent-chat/src/host/index.ts@@ -32,6 +32,8 @@ export type AgentHostConfig = {/** appended to each harness's system/developer prompt */instructions?: string;version: string;+ /** Where uploaded attachments are written; default a folder in the system temp dir. */+ uploadDir?: string;/** For tests: replaces the real harnesses. */harnesses?: Harness[];log?: (message: string) => void;@@ -56,6 +58,7 @@ export async function createAgentHost(config: AgentHostConfig): Promise<RunningAmcp: config.mcp,instructions: config.instructions,version: config.version,+ uploadDir: config.uploadDir,log,});await agentHost.start();diff --git a/packages/agent-chat/src/protocol.ts b/packages/agent-chat/src/protocol.tsindex 460822fc..739abdf9 100644--- a/packages/agent-chat/src/protocol.ts+++ b/packages/agent-chat/src/protocol.ts@@ -118,6 +118,7 @@ export type MethodMap = {result: { chatId: string };};"turn.interrupt": { params: { chatId: string }; result: void };+ "attachments.upload": { params: { name: string; data: string }; result: { path: string } };"request.respond": {params: { chatId: string; requestId: string; response: RequestResponse };result: void;diff --git a/packages/agent-chat/test/e2e.test.ts b/packages/agent-chat/test/e2e.test.tsindex 33dbd6c4..c9eb587a 100644--- a/packages/agent-chat/test/e2e.test.ts+++ b/packages/agent-chat/test/e2e.test.ts@@ -5,9 +5,9 @@// The whole loop over a real socket: the browser client against the host// with the fake harness behind it. What the UI does is what these do.-import { existsSync, mkdtempSync, rmSync } from "node:fs";+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";import { tmpdir } from "node:os";-import { join } from "node:path";+import { dirname, join } from "node:path";import { WebSocket } from "ws";import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";@@ -37,6 +37,7 @@ beforeAll(async () => {mcp: { name: "diffusion", url: "http://127.0.0.1:3274/mcp?client=chat" },version: "0.0.0-test",harnesses: [fake],+ uploadDir: join(dir, "uploads"),allowedOrigins: ["null"],log: () => {},});@@ -306,6 +307,25 @@ describe("agent chat end to end", () => {expect(existsSync(join(dir, "chats", kept))).toBe(true);});+ it("writes an uploaded attachment to a path of its own, under its name", async () => {+ const client = await connect();+ const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 1, 2, 255]);+ const data = bytes.toString("base64");++ const first = await client.request("attachments.upload", { name: "image.png", data });+ const second = await client.request("attachments.upload", { name: "image.png", data });+ expect(first.path).not.toBe(second.path);+ for (const { path } of [first, second]) {+ expect(path.endsWith("image.png")).toBe(true);+ expect(readFileSync(path)).toEqual(bytes);+ }++ // A name is only ever a name: it cannot climb out of the upload folder.+ const sneaky = await client.request("attachments.upload", { name: "../../.evil.png", data });+ expect(dirname(dirname(sneaky.path))).toBe(join(dir, "uploads"));+ expect(sneaky.path.endsWith("evil.png")).toBe(true);+ });+it("refuses a harness that is not ready", async () => {const client = await connect();fake.probeStatus = "not-installed";