The VS Code of video editing, built for agents.
TypeScript
feat: implement clipboard functionality for media import and enhance context menu options
konstantin-paulus893eab6d02a0
diff --git a/apps/desktop/src/clipboard.ts b/apps/desktop/src/clipboard.tsnew file mode 100644index 00000000..0baf326b--- /dev/null+++ b/apps/desktop/src/clipboard.ts@@ -0,0 +1,47 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The files on the system clipboard: what a copy in Finder or Explorer (or+// of a recording in a screenshot app) leaves there. The renderer reads+// pictures itself, but the web clipboard API knows nothing of files, and+// outside a paste event this is the only way to them.++import { clipboard } from "electron";+import { fileURLToPath } from "node:url";++/** The files on the clipboard, by path, in the order they were copied. */+export function clipboardPaths(): string[] {+ switch (process.platform) {+ case "darwin": {+ // Every file of a copy is in the filenames list, a plist of paths; the+ // file URL type only holds the first.+ const plist = clipboard.read("NSFilenamesPboardType");+ const paths = [...plist.matchAll(/<string>([^<]*)<\/string>/g)].map((match) => unescapeXml(match[1]!));+ if (paths.length) return paths;+ const url = clipboard.read("public.file-url");+ return url.startsWith("file:") ? [fileURLToPath(url)] : [];+ }+ case "win32": {+ // Only the first file of a copy: the full list is in CF_HDROP, which+ // Electron cannot read.+ const path = clipboard.readBuffer("FileNameW").toString("utf16le").split("\0")[0];+ return path ? [path] : [];+ }+ default:+ return clipboard+ .read("text/uri-list")+ .split(/\r?\n/)+ .filter((line) => line.startsWith("file:"))+ .map((url) => fileURLToPath(url));+ }+}++function unescapeXml(text: string): string {+ return text+ .replace(/</g, "<")+ .replace(/>/g, ">")+ .replace(/"/g, '"')+ .replace(/'/g, "'")+ .replace(/&/g, "&");+}diff --git a/apps/desktop/src/main-channels.ts b/apps/desktop/src/main-channels.tsindex 86c36918..ac245032 100644--- a/apps/desktop/src/main-channels.ts+++ b/apps/desktop/src/main-channels.ts@@ -42,6 +42,7 @@ export const MAIN_CHANNELS = {WINDOW_CAPTURE: "window:capture",WINDOW_SET_BUSY: "window:set-busy",WINDOW_SHOW: "window:show",+ CLIPBOARD_PATHS: "clipboard:paths",FILE_TRANSFER: "file:transfer",FILE_WRITE_OPEN: "file:write-open",FILE_WRITE_CHUNK: "file:write-chunk",@@ -208,6 +209,7 @@ export type MainRequestMap = {[MAIN_CHANNELS.WINDOW_CAPTURE]: { request: void; response: ScreenshotResult };[MAIN_CHANNELS.WINDOW_SET_BUSY]: { request: { busy: boolean }; response: void };[MAIN_CHANNELS.WINDOW_SHOW]: { request: void; response: void };+ [MAIN_CHANNELS.CLIPBOARD_PATHS]: { request: void; response: string[] };[MAIN_CHANNELS.FILE_TRANSFER]: {request: { selector: string; absolutePath: string };response: void;diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.tsindex 810c590e..a52a327b 100644--- a/apps/desktop/src/main.ts+++ b/apps/desktop/src/main.ts@@ -16,6 +16,7 @@ import { cliStatus, installCli, refreshCliShim, uninstallCli } from "./cli-instaimport { applyMcp, healMcpRegistrations, mcpStatus } from "./mcp-install";import { configureAnalytics, countCanvasEdits, startAnalytics, track } from "./analytics";import { setupAppMenu } from "./menu";+import { clipboardPaths } from "./clipboard";import { installedEditors, openInEditor } from "./editors";import { AppTray } from "./tray";import { WindowHost } from "./window-host";@@ -380,6 +381,7 @@ if (squirrelLaunch) {// A plain Uint8Array over the PNG, so the renderer sees bytes and not a Buffer.return { png: new Uint8Array(png.buffer, png.byteOffset, png.byteLength), width, height };});+ mainBridge.handle(MAIN_CHANNELS.CLIPBOARD_PATHS, () => clipboardPaths());mainBridge.handle(MAIN_CHANNELS.LOGS_GET, () => logBuffer);mainBridge.handle(MAIN_CHANNELS.AGENT_CHAT_ENDPOINT, () => agentChatEndpoint());mainBridge.handle(MAIN_CHANNELS.MCP_STATUS, () => mcpStatus());diff --git a/apps/web/src/components/canvas/canvas.tsx b/apps/web/src/components/canvas/canvas.tsxindex 970226cc..ad62c035 100644--- a/apps/web/src/components/canvas/canvas.tsx+++ b/apps/web/src/components/canvas/canvas.tsx@@ -107,9 +107,9 @@ export function Canvas() {<Toolbar /><CanvasCommands /><DesktopAppBanner />- <SceneInitOverlay /><NodeContextMenu class="size-full" pick={(event) => nodesAt(world, event)}><EngineCanvas />+ <SceneInitOverlay /></NodeContextMenu><CameraController /></div>diff --git a/apps/web/src/components/commands/shortcuts.tsx b/apps/web/src/components/commands/shortcuts.tsxindex 773b458d..4ab9ec01 100644--- a/apps/web/src/components/commands/shortcuts.tsx+++ b/apps/web/src/components/commands/shortcuts.tsx@@ -18,6 +18,7 @@ import {togglePlayback,} from "@diffusionstudio/runtime";import { useCommand } from "@/context/commands";+import { clipboardHasMedia } from "@/lib/clipboard";import {copySelection,cutSelection,@@ -337,7 +338,7 @@ const SHORTCUTS: Shortcut[] = [keys: ["⌘", "V"],keywords: ["clipboard"],run: pasteSelection,- available: ({ editor }) => editor.canPaste(),+ available: ({ editor }) => editor.canPaste() || clipboardHasMedia(),},{id: "duplicate",diff --git a/apps/web/src/components/sidebar-left/assets.tsx b/apps/web/src/components/sidebar-left/assets.tsxindex 9813b845..a220716f 100644--- a/apps/web/src/components/sidebar-left/assets.tsx+++ b/apps/web/src/components/sidebar-left/assets.tsx@@ -40,7 +40,8 @@ import { LazyAssetItem } from "./asset-item";import { FolderItem, handleFolderDrop, isAssetOrFolderDrag, ASSET_DRAG_TYPE, FOLDER_DRAG_TYPE } from "./folder-item";import { useLibrary } from "@/engine/library";import { useAssetSelection } from "@/engine/hooks";-import { droppedFiles, importFiles, insertAssetsAtPlayhead, insertAssetsAtStart, pickAndImport } from "@/engine/asset-actions";+import { droppedFiles, importClipboard, importFiles, insertAssetsAtPlayhead, insertAssetsAtStart, pickAndImport } from "@/engine/asset-actions";+import { clipboardHasMedia, hasMedia, pastedMedia, readClipboardMedia } from "@/lib/clipboard";import { isInputTarget } from "@/utils";import { useCommand } from "@/context/commands";@@ -330,6 +331,25 @@ export function Assets() {const handleImportAssets = () => withLibrary(async (lib) => selectImported(await pickAndImport(lib, currentFolder())));+ /** Media copied in another app, into the folder in view: from the menu. */+ const handlePaste = () => withLibrary(async (lib) => {+ selectImported(await importClipboard(lib, await readClipboardMedia(), currentFolder()));+ });++ /**+ * ⌘V with the panel in focus. Without media copied in another app, the+ * paste goes on to the stage, as it would with the panel out of focus.+ */+ const handlePasteEvent = (event: ClipboardEvent) => {+ if (isInputTarget(event)) return;+ const media = pastedMedia(event);+ const lib = library();+ if (!hasMedia(media) || !lib) return;+ event.preventDefault();+ event.stopPropagation();+ importClipboard(lib, media, currentFolder()).then(selectImported);+ };+const handleDrop = async (event: DragEvent) => {event.preventDefault();event.stopPropagation();@@ -557,6 +577,7 @@ export function Assets() {tabIndex={0}ref={root}onKeyDown={handleKeyDown}+ on:paste={handlePasteEvent}><div class="h-12 shrink-0 flex items-center gap-2 px-4 border-y border-border [[data-platform=win32]_&]:border-t-0"><div class="flex-1 min-w-0 flex items-center">@@ -788,6 +809,10 @@ export function Assets() {<ContextMenuItem onSelect={() => openPromptInput(restoreConfig("IMAGE"))}>Generate with AI</ContextMenuItem>+ <ContextMenuItem disabled={!clipboardHasMedia()} onSelect={handlePaste}>+ Paste+ <ContextMenuShortcut>⌘V</ContextMenuShortcut>+ </ContextMenuItem><ContextMenuSeparator /><ContextMenuItem onSelect={handleCreateFolder}>Create folderdiff --git a/apps/web/src/engine/asset-actions.ts b/apps/web/src/engine/asset-actions.tsindex 46b06f3f..a6618ab6 100644--- a/apps/web/src/engine/asset-actions.ts+++ b/apps/web/src/engine/asset-actions.ts@@ -12,10 +12,12 @@ import { insertAsset } from "./insert-asset";import { insertAssetsInNewScene } from "./new-scene";import { forgetAssetMedia } from "./timeline/media";import { forgetAssetPeaks } from "./timeline/peaks";+import { mimeTypeToExtension } from "@/utils/media";-import type { World } from "koota";+import type { Entity, World } from "koota";-import type { Asset, AssetLibrary } from "@diffusionstudio/assets";+import type { Asset, AssetLibrary, ImportResult } from "@diffusionstudio/assets";+import type { ClipboardMedia } from "@/lib/clipboard";export { droppedFiles, pickFiles } from "@diffusionstudio/assets";@@ -35,10 +37,48 @@ export async function importFiles(library: AssetLibrary, files: ReadonlyArray<Fiif (report.unnamed.length) {toast("Some files could not be imported", { description: "Only files on this computer can be added to the library." });}- for (const { source, error } of report.failed) {+ return reported(report);+}++/** What an import brought in, with whatever it refused reported. */+function reported(result: ImportResult): Asset[] {+ for (const { source, error } of result.failed) {toast.error(`Could not import ${source.split(/[\\/]/).pop()}`, { description: error.message });}- return report.assets;+ return result.assets;+}++/**+ * Brings what the clipboard holds into the library at `folder`: copied files+ * are linked where they are, as a drop links them; a picture has no file of+ * its own, so it is written into the project.+ */+export async function importClipboard(library: AssetLibrary, media: ClipboardMedia, folder: string): Promise<Asset[]> {+ const assets = media.paths.length ? reported(await library.import(media.paths, { folder })) : [];+ for (const image of media.images) {+ try {+ assets.push(await library.store(image, { folder, name: `Pasted image${mimeTypeToExtension(image.type)}` }));+ } catch (error) {+ toast.error("Could not paste the image", { description: (error as Error).message });+ }+ }+ return assets;+}++/**+ * Pastes what the clipboard holds onto the stage: into the library first,+ * then into `parent`, or the active scene (a scene of their own when there+ * is none), at the playhead.+ */+export async function pasteClipboardMedia(world: World, library: AssetLibrary, media: ClipboardMedia, parent?: Entity): Promise<void> {+ const assets = await importClipboard(library, media, "");+ if (!parent) return insertAssets(world, assets);+ for (const asset of assets) {+ if (!insertAsset(world, asset, { parent })) {+ toast("Nothing to insert into", { description: "Open a project first." });+ return;+ }+ }}/** Opens the file picker and imports what the user picks into `folder`. */diff --git a/apps/web/src/engine/create-engine.ts b/apps/web/src/engine/create-engine.tsindex 3a0bafa0..b5584b2b 100644--- a/apps/web/src/engine/create-engine.ts+++ b/apps/web/src/engine/create-engine.ts@@ -9,7 +9,8 @@ import { AssetSelection, DrawTool, GradientTool, Hud, Keys, MODIFIER_KEYS, Objecimport { inputSystem } from './input/input-system';import { clearClipFrames, clearClipPeaks, clearMedia, clearPeaks, timelineSystem, TimelineSurface } from './timeline';import { clearObjectTrackOf, clearObjectTracks } from './object-mask';-import { shortcutSystem } from './input/shortcuts';+import { pasteInto, shortcutSystem } from './input/shortcuts';+import { pastedMedia } from '@/lib/clipboard';import type { RuntimeWorld } from '@diffusionstudio/runtime';import type { CanvasPointerEvent, PointerEventType } from '@diffusionstudio/runtime';@@ -133,7 +134,27 @@ class Engine {if (isMod) keys.pressed.add('mod');}+ // ⌘V goes on to the browser+ if (key === 'v' && (event.metaKey || event.ctrlKey)) return;++ event.preventDefault();+ }++ /**+ * The paste ⌘V makes, with what it carries read while the event lasts.+ * Pastes into text fields belong to the field, and so do those a+ * composer took for itself.+ */+ private readonly onPaste = (event: ClipboardEvent) => {+ if (+ event.defaultPrevented+ || event.target instanceof HTMLInputElement+ || event.target instanceof HTMLTextAreaElement+ || (event.target instanceof HTMLElement && event.target.isContentEditable)+ ) return;+event.preventDefault();+ pasteInto(this.world, pastedMedia(event));}private readonly onKeyUp = (event: KeyboardEvent) => {@@ -253,6 +274,7 @@ class Engine {window.addEventListener('pointercancel', this.onPointerUp);window.addEventListener('keydown', this.onKeyDown);window.addEventListener('keyup', this.onKeyUp);+ window.addEventListener('paste', this.onPaste);window.addEventListener('blur', this.onBlur);this.resizeObserver.observe(canvas.parentElement!);@@ -305,6 +327,7 @@ class Engine {window.removeEventListener('pointercancel', this.onPointerUp);window.removeEventListener('keydown', this.onKeyDown);window.removeEventListener('keyup', this.onKeyUp);+ window.removeEventListener('paste', this.onPaste);window.removeEventListener('blur', this.onBlur);this.resizeObserver.disconnect();}diff --git a/apps/web/src/engine/input/shortcuts.ts b/apps/web/src/engine/input/shortcuts.tsindex 64a17b29..ec35581f 100644--- a/apps/web/src/engine/input/shortcuts.ts+++ b/apps/web/src/engine/input/shortcuts.ts@@ -27,6 +27,7 @@ import {getSelection,isGroupLike,isSequence,+ Library,setPlayhead,shuttlePlayback,stopPlayback,@@ -35,6 +36,8 @@ import {} from '@diffusionstudio/runtime';import { Not, Or } from 'koota';+import { claimClipboard, hasMedia, readClipboardMedia } from '@/lib/clipboard';+import { pasteClipboardMedia } from '../asset-actions';import { zoomBy, zoomTo, zoomToFit, zoomToSelection } from '../camera';import { applyClipPaths, cancelClipPath } from '../clip-path';import { getDocumentEditor } from '../editor';@@ -47,6 +50,7 @@ import { splitAtPlayhead } from '../split';import { Keys, MODIFIER_KEYS, ObjectMaskTool, Pointer } from '../traits';import { editTransform } from './interactions';+import type { ClipboardMedia } from '@/lib/clipboard';import type { TransformWrite } from './interactions';import type { CameraMatrix } from '@diffusionstudio/runtime';import type { Entity, World } from 'koota';@@ -89,6 +93,7 @@ export function copySelection(world: World): void {if (selected.length) {getDocumentEditor(world).copy(selected);+ claimClipboard();}}@@ -98,31 +103,53 @@ export function cutSelection(world: World): void {deleteSelection(world);}+/**+ * Pastes what the clipboard holds, from a menu or the palette: see+ * `pasteInto`. ⌘V pastes from its paste event instead (see the engine).+ */+export async function pasteSelection(world: World): Promise<void> {+ await pasteInto(world, await readClipboardMedia());+}+/*** Pastes where the selection points: into a selected container (a scene or* group, on top of its children), on top of a selected leaf in that leaf's* parent, or, with nothing selected, into the active scene. The editor keeps* a paste out of the sequence it was copied from.+ *+ * `media` copied in another app since the editor's last copy (a screenshot,+ * files from the Finder) is pasted instead: imported into the library, then+ * inserted into the same parent, or a new scene when there is none.*/-export function pasteSelection(world: World): void {- const editor = getDocumentEditor(world);+export async function pasteInto(world: World, media: ClipboardMedia): Promise<void> {const [selected] = world.query(Selected);if (selected === undefined) {- const active = getActiveEntity(world);- if (active) editor.paste(active);+ const active = getActiveEntity(world) ?? undefined;+ if (hasMedia(media)) await pasteMedia(world, media, active);+ else if (active) getDocumentEditor(world).paste(active);return;}if (isGroupLike(selected)) {- editor.paste(selected);+ if (hasMedia(media)) await pasteMedia(world, media, selected);+ else getDocumentEditor(world).paste(selected);return;}const parent = getParentEntity(selected);if (parent === null) return;+ if (hasMedia(media)) {+ await pasteMedia(world, media, parent);+ return;+ }const siblings = getEntityChildren(world, parent);- editor.paste(parent, siblings[siblings.indexOf(selected) + 1]);+ getDocumentEditor(world).paste(parent, siblings[siblings.indexOf(selected) + 1]);+}++async function pasteMedia(world: World, media: ClipboardMedia, parent: Entity | undefined): Promise<void> {+ const library = world.get(Library);+ if (library) await pasteClipboardMedia(world, library, media, parent);}const NUDGE = 1;@@ -514,7 +541,6 @@ const PRESSED_SHORTCUTS: readonly Shortcut[] = [{ keys: ['enter', 'mod', 'alt', 'shift'], action: unwrapSequenceSelection },{ keys: ['b', 'mod'], action: splitAtPlayhead },{ keys: ['c', 'mod'], action: copySelection },- { keys: ['v', 'mod'], action: pasteSelection },{ keys: ['x', 'mod'], action: cutSelection },{ keys: ['h', 'mod', 'shift'], action: toggleSelectionHidden },{ keys: ['a', 'mod'], action: selectAll },diff --git a/apps/web/src/lib/clipboard.ts b/apps/web/src/lib/clipboard.tsnew file mode 100644index 00000000..7400cb7c--- /dev/null+++ b/apps/web/src/lib/clipboard.ts@@ -0,0 +1,142 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The system clipboard, for media copied in other apps: files copied in the+// Finder (or a recording a screenshot app put there) and pictures (a+// screenshot, an image copied from a page). A ⌘V brings both in its paste+// event. A paste from a menu has no event: it reads pictures through the web+// clipboard API, which has no files, so those come from the desktop main+// process, and off the desktop a menu can only paste pictures.++import { createSignal } from "solid-js";+import { MAIN_CHANNELS } from "@desktop/main-channels";+import { mainBridge } from "@/lib/ipc";++/**+ * Media to paste: files that are on this computer, by path, and pictures+ * that are only bytes. Copied files come alone: a copied file reads as a+ * picture of itself too, and the file is what was copied.+ */+export type ClipboardMedia = { paths: string[]; images: Blob[] };++const NONE: ClipboardMedia = { paths: [], images: [] };++export const hasMedia = (media: ClipboardMedia) => media.paths.length > 0 || media.images.length > 0;++/**+ * What a paste event carries. Rich text from a document app carries a+ * picture of itself alongside, and that is not what was copied.+ */+export function pastedMedia(event: ClipboardEvent): ClipboardMedia {+ const data = event.clipboardData;+ if (!data || data.types.includes("text/rtf")) return NONE;++ const paths: string[] = [];+ const images: Blob[] = [];+ for (const file of Array.from(data.files)) {+ const path = window.desktop?.getPathForFile(file) || null;+ if (path) paths.push(path);+ else if (file.type.startsWith("image/")) images.push(file);+ }+ return paths.length ? { paths, images: [] } : { paths, images };+}++/** What the clipboard holds to paste, read without a paste event; nothing when it cannot be read. */+export async function readClipboardMedia(): Promise<ClipboardMedia> {+ const paths = await readClipboardPaths();+ if (paths.length) {+ setMediaOnClipboard(true);+ return { paths, images: [] };+ }++ const items = await readClipboardItems();+ const images = await Promise.all(+ items.flatMap((item) => {+ const type = imageType(item);+ return type ? [item.getType(type)] : [];+ }),+ );+ setMediaOnClipboard(images.length > 0);+ return { paths, images };+}++async function readClipboardPaths(): Promise<string[]> {+ if (!window.desktop) return [];+ try {+ return await mainBridge.call(MAIN_CHANNELS.CLIPBOARD_PATHS, undefined);+ } catch (error) {+ console.warn("[clipboard] could not read the copied files", error);+ return [];+ }+}++async function readClipboardItems(): Promise<ClipboardItem[]> {+ try {+ return await navigator.clipboard.read();+ } catch {+ // Not allowed, or the window is not focused.+ return [];+ }+}++/**+ * The picture's type in an item. The web clipboard API shows no rich text,+ * so plain text alongside is what gives away a document app's picture of it.+ */+function imageType(item: ClipboardItem): string | undefined {+ if (item.types.includes("text/plain")) return undefined;+ return item.types.find((type) => type.startsWith("image/"));+}++/**+ * Empties the clipboard of what other apps put there, for a copy the editor+ * keeps itself: a paste after it pastes that copy, not the screenshot taken+ * before it.+ */+export function claimClipboard(): void {+ setMediaOnClipboard(false);+ navigator.clipboard?.writeText("").catch((error) => console.warn("[clipboard] could not claim the clipboard", error));+}++const [mediaOnClipboard, setMediaOnClipboard] = createSignal(false);+let watching = false;++/**+ * Whether the clipboard holds media to paste, for a menu to offer Paste by.+ * Looked at again whenever the window comes back into focus or the pointer+ * back over it (the copy was made in another app, perhaps in a panel that+ * never took focus) and when a context menu opens, so it is reactive but a+ * beat behind the clipboard: a paste reads the clipboard itself. Only+ * looked at once reading is allowed: in a browser, a look would otherwise+ * ask the user for it out of the blue.+ */+export function clipboardHasMedia(): boolean {+ if (!watching) {+ watching = true;+ window.addEventListener("focus", look);+ document.documentElement.addEventListener("mouseenter", look);+ window.addEventListener("contextmenu", look, true);+ void look();+ }+ return mediaOnClipboard();+}++async function look(): Promise<void> {+ if (!(await mayRead())) return;+ if ((await readClipboardPaths()).length) {+ setMediaOnClipboard(true);+ return;+ }+ setMediaOnClipboard((await readClipboardItems()).some((item) => imageType(item) !== undefined));+}++async function mayRead(): Promise<boolean> {+ if (window.desktop) return true;+ try {+ const status = await navigator.permissions.query({ name: "clipboard-read" as PermissionName });+ return status.state === "granted";+ } catch {+ return false;+ }+}diff --git a/apps/web/src/utils/browser.ts b/apps/web/src/utils/browser.tsindex 4429a312..5bdfe732 100644--- a/apps/web/src/utils/browser.ts+++ b/apps/web/src/utils/browser.ts@@ -151,7 +151,7 @@ export async function downloadObject(source: Blob, name?: string): Promise<void>a.remove();}-export function isInputTarget(e: KeyboardEvent) {+export function isInputTarget(e: Event) {const target = e.target as HTMLElement;return target.tagName === 'INPUT' || target.tagName === 'TEXTAREA' || target.isContentEditable;}