Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add file attachment handling and paste support in agent chat

konstantin-paulus9bde5fe6ef5f
diff --git a/apps/web/src/agent-chat/attachments.tsx b/apps/web/src/agent-chat/attachments.tsx
index e0832379..6eb98152 100644
--- a/apps/web/src/agent-chat/attachments.tsx
+++ b/apps/web/src/agent-chat/attachments.tsx
@@ -4,6 +4,7 @@
import { Show, createSignal } from "solid-js";
+import { toast } from "somoto";
import { pickFiles } from "@diffusionstudio/assets";
@@ -11,17 +12,21 @@ import { Icon } from "@/components/ui/icon";
import { RemoveButton } from "@/components/ui/remove-button";
import { cx } from "@/lib/cva";
+import { client, hasHost } from "./connection";
+
/**
* A file or folder dropped onto a composer. Only what the tile and the
* handoff need: the name to label it, the kind and extension to draw it, and
* the path to send — null off the desktop, where the browser will not say
- * where a dropped file lives.
+ * where a dropped file lives. An image picked, dropped or pasted also carries
+ * an object URL for its thumbnail; one rebuilt from a path does not.
*/
export type Attachment = {
key: string;
name: string;
kind: "file" | "folder";
path: string | null;
+ preview?: string;
};
/** The paths worth sending: attachments the shell could locate. */
@@ -57,9 +62,66 @@ export function droppedAttachments(event: DragEvent): Attachment[] {
}
/** A file picked or dropped without the entry API, which only a drop has. */
-function fileAttachment(file: File, kind: Attachment["kind"] = "file", name = file.name): Attachment {
- const path = window.desktop?.getPathForFile(file) || null;
- return { key: path ?? `${name}:${file.size}:${file.lastModified}`, name, kind, path };
+function fileAttachment(file: File, kind: Attachment["kind"] = "file", name = file.name, path = window.desktop?.getPathForFile(file) || null): Attachment {
+ const preview = kind === "file" && file.type.startsWith("image/") ? URL.createObjectURL(file) : undefined;
+ return { key: path ?? `${name}:${file.size}:${file.lastModified}`, name, kind, path, ...(preview ? { preview } : {}) };
+}
+
+/**
+ * The files on the clipboard. None when the paste is text: rich text from a
+ * document app carries a picture of itself alongside, and that is not what
+ * the user meant to paste.
+ */
+function pastedFiles(event: ClipboardEvent): File[] {
+ const data = event.clipboardData;
+ if (!data || data.types.includes("text/rtf")) return [];
+ return Array.from(data.files);
+}
+
+/**
+ * Pasted files as attachments. A file copied in the Finder has a path and is
+ * sent as it is; a screenshot or an image copied from a page is only bytes,
+ * so it is uploaded to the agent host, which may not share this disk, and
+ * the path it was written to is what gets sent.
+ */
+async function pastedAttachments(files: File[]): Promise<Attachment[]> {
+ return Promise.all(
+ files.map(async (file) => {
+ const path = window.desktop?.getPathForFile(file) || null;
+ if (path) return file.type.startsWith("image/") ? fileAttachment(file, "file", file.name, path) : attachmentFromPath(path);
+ if (!hasHost()) return fileAttachment(file);
+ const saved = await client.request("attachments.upload", { name: file.name, data: await base64(file) });
+ return fileAttachment(file, "file", file.name, saved.path);
+ }),
+ );
+}
+
+/** A file's bytes as base64, without the data URL's prefix. */
+function base64(file: File): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => {
+ const url = reader.result as string;
+ resolve(url.slice(url.indexOf(",") + 1));
+ };
+ reader.onerror = () => reject(reader.error);
+ reader.readAsDataURL(file);
+ });
+}
+
+/**
+ * A paste handler for a composer: files go to `onPaste`, text is left to the
+ * textarea. A paste that fails to save is a toast, not a lost image.
+ */
+export function createPasteHandler(onPaste: (pasted: Attachment[]) => void) {
+ return (event: ClipboardEvent) => {
+ const files = pastedFiles(event);
+ if (files.length === 0) return;
+ event.preventDefault();
+ pastedAttachments(files).then(onPaste, (error: Error) =>
+ toast.error("Could not paste the file", { description: error.message }),
+ );
+ };
}
/** Files chosen from the system picker, as attachments; folders can only be dropped. */
@@ -151,9 +213,9 @@ type AttachmentTileProps = {
};
/**
- * One dropped file or folder: a grey square with a folder mark, or the file's
- * type in the middle. There is no thumbnail to show — nothing is loaded — so
- * the name is in the tooltip and the remove button appears on hover, as it
+ * One attached file or folder: the image itself when there is one to show,
+ * else a grey square with a folder mark, or the file's type in the middle.
+ * The name is in the tooltip and the remove button appears on hover, as it
* does on the generation composer's reference images.
*/
export function AttachmentTile(props: AttachmentTileProps) {
@@ -164,14 +226,21 @@ export function AttachmentTile(props: AttachmentTileProps) {
>
<div class="grid size-full place-items-center overflow-hidden rounded-lg bg-input text-muted-foreground">
<Show
- when={props.attachment.kind === "folder"}
+ when={props.attachment.preview}
fallback={
- <span class="max-w-9 truncate px-0.5 text-[9px] font-450 uppercase tracking-wide">
- {fileType(props.attachment.name)}
- </span>
+ <Show
+ when={props.attachment.kind === "folder"}
+ fallback={
+ <span class="max-w-9 truncate px-0.5 text-[9px] font-450 uppercase tracking-wide">
+ {fileType(props.attachment.name)}
+ </span>
+ }
+ >
+ <Icon name="navigation.folder" class="size-6" />
+ </Show>
}
>
- <Icon name="navigation.folder" class="size-6" />
+ {(src) => <img src={src()} alt="" draggable={false} class="size-full object-cover" />}
</Show>
</div>
<RemoveButton
diff --git a/apps/web/src/agent-chat/composer.tsx b/apps/web/src/agent-chat/composer.tsx
index 322cfda3..a3e05a41 100644
--- a/apps/web/src/agent-chat/composer.tsx
+++ b/apps/web/src/agent-chat/composer.tsx
@@ -13,7 +13,7 @@ import { Icon } from "@/components/ui/icon";
import type { ModelRef } from "@diffusionstudio/agent-chat";
-import { AttachmentTile, DropOverlay, createDropZone, mergeAttachments, pickAttachments, type Attachment } from "./attachments";
+import { AttachmentTile, DropOverlay, createDropZone, createPasteHandler, mergeAttachments, pickAttachments, type Attachment } from "./attachments";
import { ModelPicker } from "./model-picker";
const MAX_HEIGHT_PX = 160;
@@ -40,6 +40,7 @@ export function Composer(props: ComposerProps) {
const attach = (added: Attachment[]) => props.onAttachments(mergeAttachments(props.attachments, added));
const drop = createDropZone(attach);
+ const paste = createPasteHandler(attach);
const canSend = () => !props.running && !props.blocked && props.model !== null && (props.text.trim().length > 0 || props.attachments.length > 0);
@@ -73,6 +74,7 @@ export function Composer(props: ComposerProps) {
onDragEnter={drop.onDragEnter}
onDragLeave={drop.onDragLeave}
onDrop={drop.onDrop}
+ onPaste={paste}
>
<Show when={props.attachments.length > 0}>
{/* The remove buttons overhang the tiles' corners, and a scrolling
diff --git a/apps/web/src/agent-chat/index.ts b/apps/web/src/agent-chat/index.ts
index c2126d3a..fb33be2f 100644
--- a/apps/web/src/agent-chat/index.ts
+++ b/apps/web/src/agent-chat/index.ts
@@ -26,6 +26,7 @@ export {
DropOverlay,
attachmentPaths,
createDropZone,
+ createPasteHandler,
droppedAttachments,
mergeAttachments,
pickAttachments,
diff --git a/apps/web/src/components/dashboard/home-view.tsx b/apps/web/src/components/dashboard/home-view.tsx
index dc6f55a0..310883a4 100644
--- a/apps/web/src/components/dashboard/home-view.tsx
+++ b/apps/web/src/components/dashboard/home-view.tsx
@@ -34,6 +34,7 @@ import {
ModelPicker,
attachmentPaths,
createDropZone,
+ createPasteHandler,
currentModel,
mergeAttachments,
pickAttachments,
@@ -112,6 +113,7 @@ export function DashboardHomeView() {
const attach = (added: Attachment[]) => setAttachments((current) => mergeAttachments(current, added));
const drop = createDropZone(attach);
+ const paste = createPasteHandler(attach);
let textarea: HTMLTextAreaElement | undefined;
@@ -359,6 +361,7 @@ export function DashboardHomeView() {
onDragEnter={drop.onDragEnter}
onDragLeave={drop.onDragLeave}
onDrop={drop.onDrop}
+ onPaste={paste}
>
<Show when={attachments().length > 0}>
{/* The remove buttons overhang the tiles' corners, and a
diff --git a/packages/agent-chat/src/host/host.ts b/packages/agent-chat/src/host/host.ts
index e43264e1..4baa4af5 100644
--- a/packages/agent-chat/src/host/host.ts
+++ b/packages/agent-chat/src/host/host.ts
@@ -9,6 +9,9 @@
// and they can come and go at any time without disturbing a turn.
import { randomUUID } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
import { isPersistedEvent, reduce } from "../reduce";
import { HARNESS_IDS, HARNESS_LABELS, isHarnessId, titleFor } from "../protocol";
@@ -60,6 +63,8 @@ export type AgentHostOptions = {
idleMs?: number;
/** How old a probe may be before `refresh` re-runs it. */
probeMaxAgeMs?: number;
+ /** Where uploaded attachments are written; default a folder in the system temp dir. */
+ uploadDir?: string;
log?: (message: string) => void;
};
@@ -191,6 +196,8 @@ export class AgentHost {
return this.send(connection, p as MethodParams<"turn.send">) as Promise<R>;
case "turn.interrupt":
return this.interrupt(requireString(p, "chatId")) as Promise<R>;
+ case "attachments.upload":
+ return this.upload(p as MethodParams<"attachments.upload">) as Promise<R>;
case "request.respond": {
const { chatId, requestId, response } = p as MethodParams<"request.respond">;
return this.respond(requireString({ chatId }, "chatId"), requireString({ requestId }, "requestId"), response) as R;
@@ -200,6 +207,23 @@ export class AgentHost {
}
}
+ // ---------------------------------------------------------------------
+ // Attachments
+
+ /**
+ * A folder of its own per upload keeps the file's name — `image.png` for a
+ * screenshot — without two uploads landing on the same path.
+ */
+ private async upload(params: MethodParams<"attachments.upload">): Promise<{ path: string }> {
+ const name = basename(requireString(params, "name")).replace(/^\.+/, "") || "pasted";
+ const data = requireString(params, "data");
+ const dir = join(this.options.uploadDir ?? join(tmpdir(), "diffusion-studio-pasted"), randomUUID());
+ await mkdir(dir, { recursive: true });
+ const path = join(dir, name);
+ await writeFile(path, Buffer.from(data, "base64"));
+ return { path };
+ }
+
// ---------------------------------------------------------------------
// Harnesses
diff --git a/packages/agent-chat/src/host/index.ts b/packages/agent-chat/src/host/index.ts
index 298d1b7c..6b5756c9 100644
--- a/packages/agent-chat/src/host/index.ts
+++ b/packages/agent-chat/src/host/index.ts
@@ -32,6 +32,8 @@ export type AgentHostConfig = {
/** appended to each harness's system/developer prompt */
instructions?: string;
version: string;
+ /** Where uploaded attachments are written; default a folder in the system temp dir. */
+ uploadDir?: string;
/** For tests: replaces the real harnesses. */
harnesses?: Harness[];
log?: (message: string) => void;
@@ -56,6 +58,7 @@ export async function createAgentHost(config: AgentHostConfig): Promise<RunningA
mcp: config.mcp,
instructions: config.instructions,
version: config.version,
+ uploadDir: config.uploadDir,
log,
});
await agentHost.start();
diff --git a/packages/agent-chat/src/protocol.ts b/packages/agent-chat/src/protocol.ts
index 460822fc..739abdf9 100644
--- a/packages/agent-chat/src/protocol.ts
+++ b/packages/agent-chat/src/protocol.ts
@@ -118,6 +118,7 @@ export type MethodMap = {
result: { chatId: string };
};
"turn.interrupt": { params: { chatId: string }; result: void };
+ "attachments.upload": { params: { name: string; data: string }; result: { path: string } };
"request.respond": {
params: { chatId: string; requestId: string; response: RequestResponse };
result: void;
diff --git a/packages/agent-chat/test/e2e.test.ts b/packages/agent-chat/test/e2e.test.ts
index 33dbd6c4..c9eb587a 100644
--- a/packages/agent-chat/test/e2e.test.ts
+++ b/packages/agent-chat/test/e2e.test.ts
@@ -5,9 +5,9 @@
// The whole loop over a real socket: the browser client against the host
// with the fake harness behind it. What the UI does is what these do.
-import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
import { WebSocket } from "ws";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
@@ -37,6 +37,7 @@ beforeAll(async () => {
mcp: { name: "diffusion", url: "http://127.0.0.1:3274/mcp?client=chat" },
version: "0.0.0-test",
harnesses: [fake],
+ uploadDir: join(dir, "uploads"),
allowedOrigins: ["null"],
log: () => {},
});
@@ -306,6 +307,25 @@ describe("agent chat end to end", () => {
expect(existsSync(join(dir, "chats", kept))).toBe(true);
});
+ it("writes an uploaded attachment to a path of its own, under its name", async () => {
+ const client = await connect();
+ const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 1, 2, 255]);
+ const data = bytes.toString("base64");
+
+ const first = await client.request("attachments.upload", { name: "image.png", data });
+ const second = await client.request("attachments.upload", { name: "image.png", data });
+ expect(first.path).not.toBe(second.path);
+ for (const { path } of [first, second]) {
+ expect(path.endsWith("image.png")).toBe(true);
+ expect(readFileSync(path)).toEqual(bytes);
+ }
+
+ // A name is only ever a name: it cannot climb out of the upload folder.
+ const sneaky = await client.request("attachments.upload", { name: "../../.evil.png", data });
+ expect(dirname(dirname(sneaky.path))).toBe(join(dir, "uploads"));
+ expect(sneaky.path.endsWith("evil.png")).toBe(true);
+ });
+
it("refuses a harness that is not ready", async () => {
const client = await connect();
fake.probeStatus = "not-installed";