The VS Code of video editing, built for agents.
TypeScript
feat: add media asset commands and integrate into canvas context menu
konstantin-paulus7b3e1fc5e16b
diff --git a/apps/web/src/components/commands/index.tsx b/apps/web/src/components/commands/index.tsxindex 1ed3b39c..4198b3ca 100644--- a/apps/web/src/components/commands/index.tsx+++ b/apps/web/src/components/commands/index.tsx@@ -10,6 +10,7 @@ import { ExportVideoCommand } from "./export-video";import { GenerateCommand } from "./generate";import { LayerToggleCommands } from "./layer-toggles";import { MakeVideoCommand } from "./make-video";+import { MediaAssetCommands } from "./media-asset";import { ProjectFolderCommands } from "./project-folder";import { RemoveBackgroundCommand } from "./remove-background";import { RemoveSilenceCommand } from "./remove-silence";@@ -45,6 +46,7 @@ export function CanvasCommands() {<RemoveBackgroundCommand /><AutoCaptionsCommand /><RemoveSilenceCommand />+ <MediaAssetCommands /><ExportVideoCommand /><ExportCurrentFrameCommand /><ToolCommand tool={ToolType.MOVE} />diff --git a/apps/web/src/components/commands/media-asset.tsx b/apps/web/src/components/commands/media-asset.tsxnew file mode 100644index 00000000..4b32b97f--- /dev/null+++ b/apps/web/src/components/commands/media-asset.tsx@@ -0,0 +1,134 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { createMemo, onCleanup, onMount } from "solid-js";+import { toast } from "somoto";+import { assetFolder, isUrlSource } from "@diffusionstudio/assets";+import { setSidebarTab } from "@/agent-chat";+import { useMediaSelection } from "@/components/genai/selection";+import { revealInAssets } from "@/components/sidebar-left/reveal";+import { useCommand } from "@/context/commands";+import { useLayout } from "@/context/layout";+import { useLibrary } from "@/engine/library";+import { revealPath } from "@/lib/shell";+import { isWindowsDesktop } from "@/projects";++import type { Asset } from "@diffusionstudio/assets";++/** The layer option for the library's root; no folder path starts with a slash. */+const ROOT = "/";++export function MediaAssetCommands() {+ const library = useLibrary();+ const layout = useLayout();+ const { media } = useMediaSelection();++ /** The assets the selected media shows, each once. */+ const shown = createMemo(() => [+ ...new Set(media().map((entry) => entry.asset).filter((asset) => asset !== undefined)),+ ]);++ /** Those of them in the library: what the panel lists and a folder holds. */+ const listed = createMemo(() => shown().filter((asset) => !asset.transient));++ /** The first of them that is a file here, rather than a download's URL. */+ const file = () => (window.desktop ? shown().find((asset) => !isUrlSource(asset.source)) : undefined);++ const folders = () => [...(library()?.folders() ?? [])].sort();++ const moveTo = (id: string) => {+ library()?.move(listed(), id === ROOT ? "" : id);+ };++ const showInAssets = () => {+ const assets = listed();+ if (!assets.length) return;+ const folder = assetFolder(assets[0]!);+ // Only what is in the folder that opens: the panel selects what it shows.+ const ids = assets.filter((asset) => assetFolder(asset) === folder).map((asset) => asset.id);++ if (!layout.uiVisible()) layout.toggleUI();+ setSidebarTab("assets");+ revealInAssets({ folder, ids });+ };++ const showInFinder = async (asset: Asset) => {+ try {+ await revealPath(library()?.fs.absolute?.(asset.source) ?? asset.source);+ } catch (e) {+ toast.error("Failed to show the file", { description: (e as Error).message });+ }+ };++ useCommand({+ id: "set-folder",+ title: "Set folder",+ group: "Assets",+ icon: "navigation.folder",+ keywords: ["move", "folder", "asset", "organize"],+ available: () => listed().length > 0 && folders().length > 0,+ layer: {+ placeholder: "Search for folder",+ sections: () => [{+ name: "Folders",+ options: [{ id: ROOT, title: "All assets" }, ...folders().map((folder) => ({ id: folder, title: folder }))],+ }],+ current: () => {+ const asset = listed()[0];+ return asset ? assetFolder(asset) || ROOT : undefined;+ },+ pick: moveTo,+ },+ });++ useCommand({+ id: "reveal-in-assets",+ title: "Reveal in assets",+ group: "Assets",+ icon: "assets",+ keywords: ["show", "find", "library", "asset"],+ shortcut: ["⇧", "⌘", "O"],+ available: () => listed().length > 0,+ run: showInAssets,+ });++ useCommand({+ id: "reveal-in-finder",+ title: isWindowsDesktop() ? "Reveal in Explorer" : "Reveal in Finder",+ group: "Assets",+ icon: "external-link",+ keywords: ["show", "file", "finder", "explorer", "folder"],+ shortcut: ["⌥", "⌘", "O"],+ available: () => file() !== undefined,+ run: () => {+ const asset = file();+ if (asset) void showInFinder(asset);+ },+ });++ // ⇧⌘O and ⌥⌘O, ahead of the editor, which would otherwise record the key+ // as held. By code: ⌥ turns the O into another character on a Mac.+ const onKeyDown = (event: KeyboardEvent) => {+ if (!(event.metaKey || event.ctrlKey) || event.code !== "KeyO") return;++ if (event.shiftKey && !event.altKey && listed().length > 0) {+ event.preventDefault();+ event.stopPropagation();+ showInAssets();+ return;+ }++ const asset = file();+ if (event.altKey && !event.shiftKey && asset) {+ event.preventDefault();+ event.stopPropagation();+ showInFinder(asset);+ }+ };++ onMount(() => window.addEventListener("keydown", onKeyDown, true));+ onCleanup(() => window.removeEventListener("keydown", onKeyDown, true));++ return null;+}diff --git a/apps/web/src/components/commands/node-menu.tsx b/apps/web/src/components/commands/node-menu.tsxindex 50906244..fd892001 100644--- a/apps/web/src/components/commands/node-menu.tsx+++ b/apps/web/src/components/commands/node-menu.tsx@@ -18,6 +18,9 @@ import {ContextMenuPortal,ContextMenuSeparator,ContextMenuShortcut,+ ContextMenuSub,+ ContextMenuSubContent,+ ContextMenuSubTrigger,ContextMenuTrigger,} from "@/components/ui/context-menu";import { useCommands } from "@/context/commands";@@ -25,16 +28,21 @@ import { getDocumentEditor, useSelection } from "@/engine";import type { Entity } from "koota";import type { JSX } from "solid-js";-import type { RegisteredCommand } from "@/context/commands";+import type { CommandLayer, RegisteredCommand } from "@/context/commands";/**- * What a menu lists, by command id, a rule between sections: over a- * selection, and over nothing. A command that has nothing in the selection to- * work on is left out, and a section left with none goes with it.+ * What a menu lists, a rule between sections: commands by id, and submenus+ * of them. A command that has nothing in the selection to work on is left+ * out, and a section or submenu left with none goes with it.*/+export type MenuLayout = MenuEntry[][];++export type MenuEntry = string | { submenu: string; layout: MenuLayout };++/** What a menu lists over a selection, and over nothing. */export type NodeMenuLayout = {- nodes: string[][];- empty: string[][];+ nodes: MenuLayout;+ empty: MenuLayout;};const CLIPBOARD = ["shortcut:copy", "shortcut:cut", "shortcut:paste", "shortcut:duplicate", "shortcut:delete"];@@ -45,8 +53,14 @@ const STRUCTURE = ["shortcut:wrap-in-sequence","shortcut:unwrap-sequence",];-const MEDIA = ["upscale", "remove-background", "remove-silence", "auto-captions"];-const GENERATE = ["edit-with-prompt", "make-video", "rerun", "reuse-settings"];+/** What works on the selected media, the transforms in a submenu of their own. */+const MEDIA: MenuEntry[] = [+ { submenu: "Media actions", layout: [["upscale", "remove-background", "remove-silence"]] },+ "edit-with-prompt",+ "auto-captions",+];+const GENERATE = ["make-video", "rerun", "reuse-settings"];+const ASSET = ["set-folder", "reveal-in-assets", "reveal-in-finder"];export const CANVAS_MENU: NodeMenuLayout = {nodes: [@@ -55,6 +69,7 @@ export const CANVAS_MENU: NodeMenuLayout = {["shortcut:bring-to-front", "shortcut:send-to-back", "shortcut:toggle-hidden"],MEDIA,GENERATE,+ ASSET,],empty: [["shortcut:paste", "shortcut:select-all"], ["toggle-ui", "toggle-timeline", "toggle-chat", "actions"]],};@@ -80,7 +95,7 @@ export const TIMELINE_MENU: NodeMenuLayout = {* The timeline header's: what its buttons do, and putting the timeline away* (or bringing it back), which a double-click on it does too.*/-export const TIMELINE_HEADER_MENU: string[][] = [+export const TIMELINE_HEADER_MENU: MenuLayout = [["shortcut:toggle-playback", "shortcut:split"],["toggle-timeline"],];@@ -88,12 +103,48 @@ export const TIMELINE_HEADER_MENU: string[][] = [/** Whether `command` would run over the selection rather than ask for a target. */const offered = (command: RegisteredCommand) => (command.available?.() ?? true) && (command.ready?.() ?? true);-/** `layout`'s commands that are on offer, sections left with none left out. */-function resolve(commands: RegisteredCommand[], layout: string[][]): RegisteredCommand[][] {+/** A menu entry as it is shown: a command on offer, or a submenu with some. */+type MenuItem = RegisteredCommand | Submenu;++type Submenu = { submenu: string; sections: MenuItem[][] };++const isSubmenu = (item: MenuItem): item is Submenu => "submenu" in item;++/** Each submenu entry as last resolved. */+const resolved = new WeakMap<object, Submenu>();++const sameSections = (a: MenuItem[][], b: MenuItem[][]) =>+ a.length === b.length &&+ a.every((section, i) => section.length === b[i]!.length && section.every((item, j) => item === b[i]![j]));++/** `layout`'s commands that are on offer, sections and submenus left with none left out. */+function resolve(commands: RegisteredCommand[], layout: MenuLayout): MenuItem[][] {const byId = new Map(commands.map((command) => [command.id, command]));- return layout- .map((ids) => ids.map((id) => byId.get(id)).filter((command) => command !== undefined).filter(offered))- .filter((section) => section.length > 0);++ const resolveLayout = (sections: MenuLayout): MenuItem[][] =>+ sections+ .map((entries) =>+ entries+ .map((entry): MenuItem | undefined => {+ if (typeof entry !== "string") {+ const sections = resolveLayout(entry.layout);+ if (!sections.length) return undefined;+ // The same one while what it lists is: a new one would be+ // mounted afresh, closing the submenu under the pointer.+ const last = resolved.get(entry);+ if (last && sameSections(last.sections, sections)) return last;+ const submenu = { submenu: entry.submenu, sections };+ resolved.set(entry, submenu);+ return submenu;+ }+ const command = byId.get(entry);+ return command && offered(command) ? command : undefined;+ })+ .filter((item) => item !== undefined),+ )+ .filter((section) => section.length > 0);++ return resolveLayout(layout);}export type NodeContextMenuProps = Omit<JSX.HTMLAttributes<HTMLDivElement>, "onContextMenu"> & {@@ -142,8 +193,8 @@ export function NodeContextMenu(props: NodeContextMenuProps) {}export type CommandContextMenuProps = Omit<JSX.HTMLAttributes<HTMLDivElement>, "onContextMenu"> & {- /** What the menu lists, by command id, a rule between sections. */- layout: string[][];+ /** What the menu lists, a rule between sections. */+ layout: MenuLayout;};/**@@ -169,14 +220,13 @@ export function CommandContextMenu(props: CommandContextMenuProps) {type CommandMenuProps = JSX.HTMLAttributes<HTMLDivElement> & {/** What is on offer, followed while the menu is open. */- sections(): RegisteredCommand[][];+ sections(): MenuItem[][];onContextMenu(event: MouseEvent): void;};function CommandMenu(props: CommandMenuProps) {const [local, rest] = splitProps(props, ["sections", "children"]);- const { execute } = useCommands();- const [frozen, setFrozen] = createSignal<RegisteredCommand[][] | null>(null);+ const [frozen, setFrozen] = createSignal<MenuItem[][] | null>(null);/*** Holds the list as it stands until the menu is open again: a command run@@ -196,24 +246,103 @@ function CommandMenu(props: CommandMenuProps) {</ContextMenuTrigger><ContextMenuPortal><ContextMenuContent class="w-[220px]">- <For each={sections()}>+ <MenuSections sections={sections()} onPick={freeze} />+ </ContextMenuContent>+ </ContextMenuPortal>+ </ContextMenu>+ );+}++type MenuSectionsProps = {+ sections: MenuItem[][];+ /** Before the item picked is carried out. */+ onPick(): void;+};++/** A menu's items, or a submenu's, a rule between sections. */+function MenuSections(props: MenuSectionsProps) {+ const { execute } = useCommands();++ const item = (entry: MenuItem) => {+ if (isSubmenu(entry)) {+ return (+ <ContextMenuSub>+ <ContextMenuSubTrigger>{entry.submenu}</ContextMenuSubTrigger>+ <ContextMenuPortal>+ <ContextMenuSubContent class="w-[220px]">+ <MenuSections sections={entry.sections} onPick={props.onPick} />+ </ContextMenuSubContent>+ </ContextMenuPortal>+ </ContextMenuSub>+ );+ }++ const command = entry;+ if (command.layer) return <LayerSubmenu title={command.title} layer={command.layer} onPick={props.onPick} />;++ return (+ <ContextMenuItem+ onSelect={() => {+ props.onPick();+ execute(command);+ }}+ >+ {command.title}+ <Show when={command.shortcut}>+ {(keys) => <ContextMenuShortcut>{keys().join("")}</ContextMenuShortcut>}+ </Show>+ </ContextMenuItem>+ );+ };++ return (+ <For each={props.sections}>+ {(section, index) => (+ <>+ <Show when={index() > 0}>+ <ContextMenuSeparator />+ </Show>+ <ContextMenuGroup>+ <For each={section}>{item}</For>+ </ContextMenuGroup>+ </>+ )}+ </For>+ );+}++type LayerSubmenuProps = {+ title: string;+ layer: CommandLayer;+ /** Before the option is carried out. */+ onPick(): void;+};++/** A command the palette lists options for, as a submenu of them: a rule between its sections. */+function LayerSubmenu(props: LayerSubmenuProps) {+ return (+ <ContextMenuSub>+ <ContextMenuSubTrigger>{props.title}</ContextMenuSubTrigger>+ <ContextMenuPortal>+ <ContextMenuSubContent class="w-[220px]">+ <For each={props.layer.sections()}>{(section, index) => (<><Show when={index() > 0}><ContextMenuSeparator /></Show><ContextMenuGroup>- <For each={section}>- {(command) => (+ <For each={section.options}>+ {(option) => (<ContextMenuItemonSelect={() => {- freeze();- execute(command);+ props.onPick();+ props.layer.pick(option.id);}}>- {command.title}- <Show when={command.shortcut}>- {(keys) => <ContextMenuShortcut>{keys().join("")}</ContextMenuShortcut>}+ <span class="min-w-0 flex-1 truncate">{option.title}</span>+ <Show when={option.detail}>+ {(detail) => <ContextMenuShortcut>{detail()}</ContextMenuShortcut>}</Show></ContextMenuItem>)}@@ -222,8 +351,8 @@ function CommandMenu(props: CommandMenuProps) {</>)}</For>- </ContextMenuContent>+ </ContextMenuSubContent></ContextMenuPortal>- </ContextMenu>+ </ContextMenuSub>);}diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsxindex aaf3424b..ff5f63f1 100644--- a/apps/web/src/components/genai/use-auto-captions.tsx+++ b/apps/web/src/components/genai/use-auto-captions.tsx@@ -8,14 +8,21 @@ import { useProject } from "@/context/project";import { generateCaptions } from "@/engine/generate";import { useActiveScene, useSelection } from "@/engine/hooks";-/** Captions for the selected scene, or the active one, the one being edited, when none is selected. */+/**+ * Captions for the selected scene, or the active one, the one being edited,+ * when nothing is selected. A selection without a scene in it has none: other+ * nodes are not captioned.+ */export function useAutoCaptions() {const world = useWorld();const project = useProject();const { nodes } = useSelection();const active = useActiveScene();- const scene = () => nodes().find((node) => node.has(Scene)) ?? active();+ const scene = () => {+ const selected = nodes();+ return selected.length ? selected.find((node) => node.has(Scene)) ?? null : active();+ };const generate = () => {const target = scene();diff --git a/apps/web/src/components/sidebar-left/assets.tsx b/apps/web/src/components/sidebar-left/assets.tsxindex 9f5c01ef..4d9703dc 100644--- a/apps/web/src/components/sidebar-left/assets.tsx+++ b/apps/web/src/components/sidebar-left/assets.tsx@@ -37,6 +37,7 @@ import {BreadcrumbsSeparator,} from "../ui/breadcrumbs";import { LazyAssetItem } from "./asset-item";+import { clearRevealRequest, revealRequest } from "./reveal";import { FolderItem, handleFolderDrop, isAssetOrFolderDrag, ASSET_DRAG_TYPE, FOLDER_DRAG_TYPE } from "./folder-item";import { useLibrary } from "@/engine/library";import { useAssetSelection } from "@/engine/hooks";@@ -338,6 +339,27 @@ export function Assets() {setCurrentFolder(folder);});+ // A reveal from elsewhere (the canvas menu's Reveal in assets): its folder+ // opens, unfiltered, with its assets selected and the first scrolled to.+ createEffect(() => {+ const request = revealRequest();+ if (!request) return;+ clearRevealRequest();++ untrack(() => {+ setQuery("");+ setAssetFilter("ALL");+ setCurrentFolder(request.folder);+ const keys = request.ids.map(assetKey);+ setSelection(keys);+ anchor = keys[0] ?? null;+ lead = keys.at(-1) ?? null;+ base = new Set();+ root?.focus();+ requestAnimationFrame(() => keys[0] && scrollToItem(keys[0]));+ });+ });+const withLibrary = (run: (library: AssetLibrary) => void | Promise<void>) => {const lib = library();if (!lib) {diff --git a/apps/web/src/components/sidebar-left/reveal.ts b/apps/web/src/components/sidebar-left/reveal.tsnew file mode 100644index 00000000..c6cf78f1--- /dev/null+++ b/apps/web/src/components/sidebar-left/reveal.ts@@ -0,0 +1,32 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Showing assets in the assets panel from elsewhere: the panel opens the+// folder they are in, selects them and scrolls to them. Kept here rather than+// in the panel, which is not mounted while the UI is hidden: it takes up a+// request left for it as it comes up.++import { createSignal } from "solid-js";++export type RevealRequest = {+ /** The folder to open ('' = the root). */+ folder: string;+ /** The assets in it to select. */+ ids: string[];+};++const [revealRequest, setRevealRequest] = createSignal<RevealRequest | null>(null);++/** The reveal the assets panel has yet to carry out. */+export { revealRequest };++/** Asks the assets panel to show `ids` in `folder`. */+export function revealInAssets(request: RevealRequest): void {+ setRevealRequest(request);+}++/** Marks the reveal done. */+export function clearRevealRequest(): void {+ setRevealRequest(null);+}