Skip to content

The VS Code of video editing, built for agents.

TypeScript

feat: add media asset commands and integrate into canvas context menu

konstantin-paulus7b3e1fc5e16b
diff --git a/apps/web/src/components/commands/index.tsx b/apps/web/src/components/commands/index.tsx
index 1ed3b39c..4198b3ca 100644
--- a/apps/web/src/components/commands/index.tsx
+++ b/apps/web/src/components/commands/index.tsx
@@ -10,6 +10,7 @@ import { ExportVideoCommand } from "./export-video";
import { GenerateCommand } from "./generate";
import { LayerToggleCommands } from "./layer-toggles";
import { MakeVideoCommand } from "./make-video";
+import { MediaAssetCommands } from "./media-asset";
import { ProjectFolderCommands } from "./project-folder";
import { RemoveBackgroundCommand } from "./remove-background";
import { RemoveSilenceCommand } from "./remove-silence";
@@ -45,6 +46,7 @@ export function CanvasCommands() {
<RemoveBackgroundCommand />
<AutoCaptionsCommand />
<RemoveSilenceCommand />
+ <MediaAssetCommands />
<ExportVideoCommand />
<ExportCurrentFrameCommand />
<ToolCommand tool={ToolType.MOVE} />
diff --git a/apps/web/src/components/commands/media-asset.tsx b/apps/web/src/components/commands/media-asset.tsx
new file mode 100644
index 00000000..4b32b97f
--- /dev/null
+++ b/apps/web/src/components/commands/media-asset.tsx
@@ -0,0 +1,134 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { createMemo, onCleanup, onMount } from "solid-js";
+import { toast } from "somoto";
+import { assetFolder, isUrlSource } from "@diffusionstudio/assets";
+import { setSidebarTab } from "@/agent-chat";
+import { useMediaSelection } from "@/components/genai/selection";
+import { revealInAssets } from "@/components/sidebar-left/reveal";
+import { useCommand } from "@/context/commands";
+import { useLayout } from "@/context/layout";
+import { useLibrary } from "@/engine/library";
+import { revealPath } from "@/lib/shell";
+import { isWindowsDesktop } from "@/projects";
+
+import type { Asset } from "@diffusionstudio/assets";
+
+/** The layer option for the library's root; no folder path starts with a slash. */
+const ROOT = "/";
+
+export function MediaAssetCommands() {
+ const library = useLibrary();
+ const layout = useLayout();
+ const { media } = useMediaSelection();
+
+ /** The assets the selected media shows, each once. */
+ const shown = createMemo(() => [
+ ...new Set(media().map((entry) => entry.asset).filter((asset) => asset !== undefined)),
+ ]);
+
+ /** Those of them in the library: what the panel lists and a folder holds. */
+ const listed = createMemo(() => shown().filter((asset) => !asset.transient));
+
+ /** The first of them that is a file here, rather than a download's URL. */
+ const file = () => (window.desktop ? shown().find((asset) => !isUrlSource(asset.source)) : undefined);
+
+ const folders = () => [...(library()?.folders() ?? [])].sort();
+
+ const moveTo = (id: string) => {
+ library()?.move(listed(), id === ROOT ? "" : id);
+ };
+
+ const showInAssets = () => {
+ const assets = listed();
+ if (!assets.length) return;
+ const folder = assetFolder(assets[0]!);
+ // Only what is in the folder that opens: the panel selects what it shows.
+ const ids = assets.filter((asset) => assetFolder(asset) === folder).map((asset) => asset.id);
+
+ if (!layout.uiVisible()) layout.toggleUI();
+ setSidebarTab("assets");
+ revealInAssets({ folder, ids });
+ };
+
+ const showInFinder = async (asset: Asset) => {
+ try {
+ await revealPath(library()?.fs.absolute?.(asset.source) ?? asset.source);
+ } catch (e) {
+ toast.error("Failed to show the file", { description: (e as Error).message });
+ }
+ };
+
+ useCommand({
+ id: "set-folder",
+ title: "Set folder",
+ group: "Assets",
+ icon: "navigation.folder",
+ keywords: ["move", "folder", "asset", "organize"],
+ available: () => listed().length > 0 && folders().length > 0,
+ layer: {
+ placeholder: "Search for folder",
+ sections: () => [{
+ name: "Folders",
+ options: [{ id: ROOT, title: "All assets" }, ...folders().map((folder) => ({ id: folder, title: folder }))],
+ }],
+ current: () => {
+ const asset = listed()[0];
+ return asset ? assetFolder(asset) || ROOT : undefined;
+ },
+ pick: moveTo,
+ },
+ });
+
+ useCommand({
+ id: "reveal-in-assets",
+ title: "Reveal in assets",
+ group: "Assets",
+ icon: "assets",
+ keywords: ["show", "find", "library", "asset"],
+ shortcut: ["⇧", "⌘", "O"],
+ available: () => listed().length > 0,
+ run: showInAssets,
+ });
+
+ useCommand({
+ id: "reveal-in-finder",
+ title: isWindowsDesktop() ? "Reveal in Explorer" : "Reveal in Finder",
+ group: "Assets",
+ icon: "external-link",
+ keywords: ["show", "file", "finder", "explorer", "folder"],
+ shortcut: ["⌥", "⌘", "O"],
+ available: () => file() !== undefined,
+ run: () => {
+ const asset = file();
+ if (asset) void showInFinder(asset);
+ },
+ });
+
+ // ⇧⌘O and ⌥⌘O, ahead of the editor, which would otherwise record the key
+ // as held. By code: ⌥ turns the O into another character on a Mac.
+ const onKeyDown = (event: KeyboardEvent) => {
+ if (!(event.metaKey || event.ctrlKey) || event.code !== "KeyO") return;
+
+ if (event.shiftKey && !event.altKey && listed().length > 0) {
+ event.preventDefault();
+ event.stopPropagation();
+ showInAssets();
+ return;
+ }
+
+ const asset = file();
+ if (event.altKey && !event.shiftKey && asset) {
+ event.preventDefault();
+ event.stopPropagation();
+ showInFinder(asset);
+ }
+ };
+
+ onMount(() => window.addEventListener("keydown", onKeyDown, true));
+ onCleanup(() => window.removeEventListener("keydown", onKeyDown, true));
+
+ return null;
+}
diff --git a/apps/web/src/components/commands/node-menu.tsx b/apps/web/src/components/commands/node-menu.tsx
index 50906244..fd892001 100644
--- a/apps/web/src/components/commands/node-menu.tsx
+++ b/apps/web/src/components/commands/node-menu.tsx
@@ -18,6 +18,9 @@ import {
ContextMenuPortal,
ContextMenuSeparator,
ContextMenuShortcut,
+ ContextMenuSub,
+ ContextMenuSubContent,
+ ContextMenuSubTrigger,
ContextMenuTrigger,
} from "@/components/ui/context-menu";
import { useCommands } from "@/context/commands";
@@ -25,16 +28,21 @@ import { getDocumentEditor, useSelection } from "@/engine";
import type { Entity } from "koota";
import type { JSX } from "solid-js";
-import type { RegisteredCommand } from "@/context/commands";
+import type { CommandLayer, RegisteredCommand } from "@/context/commands";
/**
- * What a menu lists, by command id, a rule between sections: over a
- * selection, and over nothing. A command that has nothing in the selection to
- * work on is left out, and a section left with none goes with it.
+ * What a menu lists, a rule between sections: commands by id, and submenus
+ * of them. A command that has nothing in the selection to work on is left
+ * out, and a section or submenu left with none goes with it.
*/
+export type MenuLayout = MenuEntry[][];
+
+export type MenuEntry = string | { submenu: string; layout: MenuLayout };
+
+/** What a menu lists over a selection, and over nothing. */
export type NodeMenuLayout = {
- nodes: string[][];
- empty: string[][];
+ nodes: MenuLayout;
+ empty: MenuLayout;
};
const CLIPBOARD = ["shortcut:copy", "shortcut:cut", "shortcut:paste", "shortcut:duplicate", "shortcut:delete"];
@@ -45,8 +53,14 @@ const STRUCTURE = [
"shortcut:wrap-in-sequence",
"shortcut:unwrap-sequence",
];
-const MEDIA = ["upscale", "remove-background", "remove-silence", "auto-captions"];
-const GENERATE = ["edit-with-prompt", "make-video", "rerun", "reuse-settings"];
+/** What works on the selected media, the transforms in a submenu of their own. */
+const MEDIA: MenuEntry[] = [
+ { submenu: "Media actions", layout: [["upscale", "remove-background", "remove-silence"]] },
+ "edit-with-prompt",
+ "auto-captions",
+];
+const GENERATE = ["make-video", "rerun", "reuse-settings"];
+const ASSET = ["set-folder", "reveal-in-assets", "reveal-in-finder"];
export const CANVAS_MENU: NodeMenuLayout = {
nodes: [
@@ -55,6 +69,7 @@ export const CANVAS_MENU: NodeMenuLayout = {
["shortcut:bring-to-front", "shortcut:send-to-back", "shortcut:toggle-hidden"],
MEDIA,
GENERATE,
+ ASSET,
],
empty: [["shortcut:paste", "shortcut:select-all"], ["toggle-ui", "toggle-timeline", "toggle-chat", "actions"]],
};
@@ -80,7 +95,7 @@ export const TIMELINE_MENU: NodeMenuLayout = {
* The timeline header's: what its buttons do, and putting the timeline away
* (or bringing it back), which a double-click on it does too.
*/
-export const TIMELINE_HEADER_MENU: string[][] = [
+export const TIMELINE_HEADER_MENU: MenuLayout = [
["shortcut:toggle-playback", "shortcut:split"],
["toggle-timeline"],
];
@@ -88,12 +103,48 @@ export const TIMELINE_HEADER_MENU: string[][] = [
/** Whether `command` would run over the selection rather than ask for a target. */
const offered = (command: RegisteredCommand) => (command.available?.() ?? true) && (command.ready?.() ?? true);
-/** `layout`'s commands that are on offer, sections left with none left out. */
-function resolve(commands: RegisteredCommand[], layout: string[][]): RegisteredCommand[][] {
+/** A menu entry as it is shown: a command on offer, or a submenu with some. */
+type MenuItem = RegisteredCommand | Submenu;
+
+type Submenu = { submenu: string; sections: MenuItem[][] };
+
+const isSubmenu = (item: MenuItem): item is Submenu => "submenu" in item;
+
+/** Each submenu entry as last resolved. */
+const resolved = new WeakMap<object, Submenu>();
+
+const sameSections = (a: MenuItem[][], b: MenuItem[][]) =>
+ a.length === b.length &&
+ a.every((section, i) => section.length === b[i]!.length && section.every((item, j) => item === b[i]![j]));
+
+/** `layout`'s commands that are on offer, sections and submenus left with none left out. */
+function resolve(commands: RegisteredCommand[], layout: MenuLayout): MenuItem[][] {
const byId = new Map(commands.map((command) => [command.id, command]));
- return layout
- .map((ids) => ids.map((id) => byId.get(id)).filter((command) => command !== undefined).filter(offered))
- .filter((section) => section.length > 0);
+
+ const resolveLayout = (sections: MenuLayout): MenuItem[][] =>
+ sections
+ .map((entries) =>
+ entries
+ .map((entry): MenuItem | undefined => {
+ if (typeof entry !== "string") {
+ const sections = resolveLayout(entry.layout);
+ if (!sections.length) return undefined;
+ // The same one while what it lists is: a new one would be
+ // mounted afresh, closing the submenu under the pointer.
+ const last = resolved.get(entry);
+ if (last && sameSections(last.sections, sections)) return last;
+ const submenu = { submenu: entry.submenu, sections };
+ resolved.set(entry, submenu);
+ return submenu;
+ }
+ const command = byId.get(entry);
+ return command && offered(command) ? command : undefined;
+ })
+ .filter((item) => item !== undefined),
+ )
+ .filter((section) => section.length > 0);
+
+ return resolveLayout(layout);
}
export type NodeContextMenuProps = Omit<JSX.HTMLAttributes<HTMLDivElement>, "onContextMenu"> & {
@@ -142,8 +193,8 @@ export function NodeContextMenu(props: NodeContextMenuProps) {
}
export type CommandContextMenuProps = Omit<JSX.HTMLAttributes<HTMLDivElement>, "onContextMenu"> & {
- /** What the menu lists, by command id, a rule between sections. */
- layout: string[][];
+ /** What the menu lists, a rule between sections. */
+ layout: MenuLayout;
};
/**
@@ -169,14 +220,13 @@ export function CommandContextMenu(props: CommandContextMenuProps) {
type CommandMenuProps = JSX.HTMLAttributes<HTMLDivElement> & {
/** What is on offer, followed while the menu is open. */
- sections(): RegisteredCommand[][];
+ sections(): MenuItem[][];
onContextMenu(event: MouseEvent): void;
};
function CommandMenu(props: CommandMenuProps) {
const [local, rest] = splitProps(props, ["sections", "children"]);
- const { execute } = useCommands();
- const [frozen, setFrozen] = createSignal<RegisteredCommand[][] | null>(null);
+ const [frozen, setFrozen] = createSignal<MenuItem[][] | null>(null);
/**
* Holds the list as it stands until the menu is open again: a command run
@@ -196,24 +246,103 @@ function CommandMenu(props: CommandMenuProps) {
</ContextMenuTrigger>
<ContextMenuPortal>
<ContextMenuContent class="w-[220px]">
- <For each={sections()}>
+ <MenuSections sections={sections()} onPick={freeze} />
+ </ContextMenuContent>
+ </ContextMenuPortal>
+ </ContextMenu>
+ );
+}
+
+type MenuSectionsProps = {
+ sections: MenuItem[][];
+ /** Before the item picked is carried out. */
+ onPick(): void;
+};
+
+/** A menu's items, or a submenu's, a rule between sections. */
+function MenuSections(props: MenuSectionsProps) {
+ const { execute } = useCommands();
+
+ const item = (entry: MenuItem) => {
+ if (isSubmenu(entry)) {
+ return (
+ <ContextMenuSub>
+ <ContextMenuSubTrigger>{entry.submenu}</ContextMenuSubTrigger>
+ <ContextMenuPortal>
+ <ContextMenuSubContent class="w-[220px]">
+ <MenuSections sections={entry.sections} onPick={props.onPick} />
+ </ContextMenuSubContent>
+ </ContextMenuPortal>
+ </ContextMenuSub>
+ );
+ }
+
+ const command = entry;
+ if (command.layer) return <LayerSubmenu title={command.title} layer={command.layer} onPick={props.onPick} />;
+
+ return (
+ <ContextMenuItem
+ onSelect={() => {
+ props.onPick();
+ execute(command);
+ }}
+ >
+ {command.title}
+ <Show when={command.shortcut}>
+ {(keys) => <ContextMenuShortcut>{keys().join("")}</ContextMenuShortcut>}
+ </Show>
+ </ContextMenuItem>
+ );
+ };
+
+ return (
+ <For each={props.sections}>
+ {(section, index) => (
+ <>
+ <Show when={index() > 0}>
+ <ContextMenuSeparator />
+ </Show>
+ <ContextMenuGroup>
+ <For each={section}>{item}</For>
+ </ContextMenuGroup>
+ </>
+ )}
+ </For>
+ );
+}
+
+type LayerSubmenuProps = {
+ title: string;
+ layer: CommandLayer;
+ /** Before the option is carried out. */
+ onPick(): void;
+};
+
+/** A command the palette lists options for, as a submenu of them: a rule between its sections. */
+function LayerSubmenu(props: LayerSubmenuProps) {
+ return (
+ <ContextMenuSub>
+ <ContextMenuSubTrigger>{props.title}</ContextMenuSubTrigger>
+ <ContextMenuPortal>
+ <ContextMenuSubContent class="w-[220px]">
+ <For each={props.layer.sections()}>
{(section, index) => (
<>
<Show when={index() > 0}>
<ContextMenuSeparator />
</Show>
<ContextMenuGroup>
- <For each={section}>
- {(command) => (
+ <For each={section.options}>
+ {(option) => (
<ContextMenuItem
onSelect={() => {
- freeze();
- execute(command);
+ props.onPick();
+ props.layer.pick(option.id);
}}
>
- {command.title}
- <Show when={command.shortcut}>
- {(keys) => <ContextMenuShortcut>{keys().join("")}</ContextMenuShortcut>}
+ <span class="min-w-0 flex-1 truncate">{option.title}</span>
+ <Show when={option.detail}>
+ {(detail) => <ContextMenuShortcut>{detail()}</ContextMenuShortcut>}
</Show>
</ContextMenuItem>
)}
@@ -222,8 +351,8 @@ function CommandMenu(props: CommandMenuProps) {
</>
)}
</For>
- </ContextMenuContent>
+ </ContextMenuSubContent>
</ContextMenuPortal>
- </ContextMenu>
+ </ContextMenuSub>
);
}
diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsx
index aaf3424b..ff5f63f1 100644
--- a/apps/web/src/components/genai/use-auto-captions.tsx
+++ b/apps/web/src/components/genai/use-auto-captions.tsx
@@ -8,14 +8,21 @@ import { useProject } from "@/context/project";
import { generateCaptions } from "@/engine/generate";
import { useActiveScene, useSelection } from "@/engine/hooks";
-/** Captions for the selected scene, or the active one, the one being edited, when none is selected. */
+/**
+ * Captions for the selected scene, or the active one, the one being edited,
+ * when nothing is selected. A selection without a scene in it has none: other
+ * nodes are not captioned.
+ */
export function useAutoCaptions() {
const world = useWorld();
const project = useProject();
const { nodes } = useSelection();
const active = useActiveScene();
- const scene = () => nodes().find((node) => node.has(Scene)) ?? active();
+ const scene = () => {
+ const selected = nodes();
+ return selected.length ? selected.find((node) => node.has(Scene)) ?? null : active();
+ };
const generate = () => {
const target = scene();
diff --git a/apps/web/src/components/sidebar-left/assets.tsx b/apps/web/src/components/sidebar-left/assets.tsx
index 9f5c01ef..4d9703dc 100644
--- a/apps/web/src/components/sidebar-left/assets.tsx
+++ b/apps/web/src/components/sidebar-left/assets.tsx
@@ -37,6 +37,7 @@ import {
BreadcrumbsSeparator,
} from "../ui/breadcrumbs";
import { LazyAssetItem } from "./asset-item";
+import { clearRevealRequest, revealRequest } from "./reveal";
import { FolderItem, handleFolderDrop, isAssetOrFolderDrag, ASSET_DRAG_TYPE, FOLDER_DRAG_TYPE } from "./folder-item";
import { useLibrary } from "@/engine/library";
import { useAssetSelection } from "@/engine/hooks";
@@ -338,6 +339,27 @@ export function Assets() {
setCurrentFolder(folder);
});
+ // A reveal from elsewhere (the canvas menu's Reveal in assets): its folder
+ // opens, unfiltered, with its assets selected and the first scrolled to.
+ createEffect(() => {
+ const request = revealRequest();
+ if (!request) return;
+ clearRevealRequest();
+
+ untrack(() => {
+ setQuery("");
+ setAssetFilter("ALL");
+ setCurrentFolder(request.folder);
+ const keys = request.ids.map(assetKey);
+ setSelection(keys);
+ anchor = keys[0] ?? null;
+ lead = keys.at(-1) ?? null;
+ base = new Set();
+ root?.focus();
+ requestAnimationFrame(() => keys[0] && scrollToItem(keys[0]));
+ });
+ });
+
const withLibrary = (run: (library: AssetLibrary) => void | Promise<void>) => {
const lib = library();
if (!lib) {
diff --git a/apps/web/src/components/sidebar-left/reveal.ts b/apps/web/src/components/sidebar-left/reveal.ts
new file mode 100644
index 00000000..c6cf78f1
--- /dev/null
+++ b/apps/web/src/components/sidebar-left/reveal.ts
@@ -0,0 +1,32 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Showing assets in the assets panel from elsewhere: the panel opens the
+// folder they are in, selects them and scrolls to them. Kept here rather than
+// in the panel, which is not mounted while the UI is hidden: it takes up a
+// request left for it as it comes up.
+
+import { createSignal } from "solid-js";
+
+export type RevealRequest = {
+ /** The folder to open ('' = the root). */
+ folder: string;
+ /** The assets in it to select. */
+ ids: string[];
+};
+
+const [revealRequest, setRevealRequest] = createSignal<RevealRequest | null>(null);
+
+/** The reveal the assets panel has yet to carry out. */
+export { revealRequest };
+
+/** Asks the assets panel to show `ids` in `folder`. */
+export function revealInAssets(request: RevealRequest): void {
+ setRevealRequest(request);
+}
+
+/** Marks the reveal done. */
+export function clearRevealRequest(): void {
+ setRevealRequest(null);
+}