Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance Object Mask Tool Functionality and UI Integration

konstantin-paulus3f38c5eb60c3
diff --git a/apps/web/src/assets/icons/object-mask.add.svg b/apps/web/src/assets/icons/object-mask.add.svg
new file mode 100644
index 00000000..8f60edf1
--- /dev/null
+++ b/apps/web/src/assets/icons/object-mask.add.svg
@@ -0,0 +1,3 @@
+<svg viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M12 5.25C15.7279 5.25 18.75 8.27208 18.75 12C18.75 15.7279 15.7279 18.75 12 18.75C8.27208 18.75 5.25 15.7279 5.25 12C5.25 8.27208 8.27208 5.25 12 5.25ZM12 6.25C8.82436 6.25 6.25 8.82436 6.25 12C6.25 15.1756 8.82436 17.75 12 17.75C15.1756 17.75 17.75 15.1756 17.75 12C17.75 8.82436 15.1756 6.25 12 6.25ZM12.0029 9C12.1346 9 12.2614 9.05272 12.3545 9.14648C12.4476 9.24025 12.5 9.3674 12.5 9.5L12.4971 11.5039L14.501 11.5068C14.6334 11.507 14.7608 11.5593 14.8545 11.6523C14.948 11.7454 15.0009 11.8715 15.001 12.0029C15.001 12.1346 14.9482 12.2614 14.8545 12.3545C14.7608 12.4475 14.6334 12.4999 14.501 12.5L12.4951 12.4971L12.4932 14.5C12.4932 14.6326 12.4408 14.7598 12.3477 14.8535C12.2545 14.9473 12.1278 15 11.9961 15C11.8646 14.9999 11.7385 14.9472 11.6455 14.8535C11.5524 14.7597 11.5 14.6326 11.5 14.5L11.502 12.4951L9.50098 12.4932C9.36851 12.4932 9.2412 12.4406 9.14746 12.3477C9.05369 12.2545 9.00098 12.1278 9.00098 11.9961C9.00106 11.8645 9.05377 11.7386 9.14746 11.6455C9.24122 11.5525 9.36843 11.5 9.50098 11.5L11.5039 11.502L11.5068 9.5C11.5068 9.36739 11.5592 9.24025 11.6523 9.14648C11.7454 9.05285 11.8714 9.00011 12.0029 9Z" fill="currentColor"/>
+</svg>
diff --git a/apps/web/src/assets/icons/object-mask.point.svg b/apps/web/src/assets/icons/object-mask.point.svg
new file mode 100644
index 00000000..06c00c17
--- /dev/null
+++ b/apps/web/src/assets/icons/object-mask.point.svg
@@ -0,0 +1,3 @@
+<svg viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M12 5.25C15.7279 5.25 18.75 8.27208 18.75 12C18.75 15.7279 15.7279 18.75 12 18.75C8.27208 18.75 5.25 15.7279 5.25 12C5.25 8.27208 8.27208 5.25 12 5.25ZM12 6.25C8.82436 6.25 6.25 8.82436 6.25 12C6.25 15.1756 8.82436 17.75 12 17.75C15.1756 17.75 17.75 15.1756 17.75 12C17.75 8.82436 15.1756 6.25 12 6.25ZM12 10C13.1043 10.0001 13.9999 10.8957 14 12C13.9999 13.1043 13.1043 13.9999 12 14C10.8956 14 10.0001 13.1044 10 12C10.0001 10.8956 10.8956 10 12 10Z" fill="currentColor"/>
+</svg>
diff --git a/apps/web/src/assets/icons/object-mask.subtract.svg b/apps/web/src/assets/icons/object-mask.subtract.svg
new file mode 100644
index 00000000..c3a8bbb8
--- /dev/null
+++ b/apps/web/src/assets/icons/object-mask.subtract.svg
@@ -0,0 +1,3 @@
+<svg viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M12 5.25C15.7279 5.25 18.75 8.27208 18.75 12C18.75 15.7279 15.7279 18.75 12 18.75C8.27208 18.75 5.25 15.7279 5.25 12C5.25 8.27208 8.27208 5.25 12 5.25ZM12 6.25C8.82436 6.25 6.25 8.82436 6.25 12C6.25 15.1756 8.82436 17.75 12 17.75C15.1756 17.75 17.75 15.1756 17.75 12C17.75 8.82436 15.1756 6.25 12 6.25ZM14.501 11.5068C14.6334 11.507 14.7608 11.5593 14.8545 11.6523C14.948 11.7454 15.0009 11.8715 15.001 12.0029C15.001 12.1346 14.9482 12.2614 14.8545 12.3545C14.7608 12.4475 14.6334 12.4999 14.501 12.5L9.50098 12.4932C9.36851 12.4932 9.2412 12.4406 9.14746 12.3477C9.05369 12.2545 9.00098 12.1278 9.00098 11.9961C9.00106 11.8645 9.05377 11.7386 9.14746 11.6455C9.24122 11.5525 9.36843 11.5 9.50098 11.5L14.501 11.5068Z" fill="currentColor"/>
+</svg>
diff --git a/apps/web/src/components/canvas/object-mask-bar.tsx b/apps/web/src/components/canvas/object-mask-bar.tsx
new file mode 100644
index 00000000..e0b41ef4
--- /dev/null
+++ b/apps/web/src/components/canvas/object-mask-bar.tsx
@@ -0,0 +1,380 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { For, Show, createResource, onMount, splitProps } from "solid-js";
+import { SAM2_MODELS, downloadSize, sam2Model } from "@diffusionstudio/sam2/models";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { Effect, Name, getParentNode } from "@diffusionstudio/runtime";
+import { Button } from "@/components/ui/button";
+import { Icon } from "@/components/ui/icon";
+import { Separator } from "@/components/ui/separator";
+import { SegmentedIconTabs } from "@/components/ui/segmented-icon-tabs";
+import { SliderInput } from "@/components/ui/slider-input";
+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
+import { Popover, PopoverContent, PopoverPortal, PopoverTrigger } from "@/components/ui/popover";
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuItem,
+ DropdownMenuPortal,
+ DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu";
+import { cx } from "@/lib/cva";
+import { useDerived } from "@/engine/hooks";
+import {
+ brushRadius,
+ cancelObjectMask,
+ getMaskRestore,
+ getObjectTrack,
+ heldObjectMaskOp,
+ objectMaskMode,
+ objectMaskModel,
+ objectMaskModelLoad,
+ objectMaskOp,
+ pickObjectMaskModel,
+ preloadObjectMaskModel,
+ setBrushRadius,
+ setObjectMaskMode,
+ setObjectMaskOp,
+ targetEffect,
+ trackObjectMask,
+} from "@/engine/object-mask";
+import { effectOption } from "../sidebar-right/inspector/effect-types";
+
+import type { JSX } from "solid-js";
+import type { Sam2Model, Sam2ModelId } from "@diffusionstudio/sam2/models";
+import type { ObjectMaskMode, ObjectMaskModelLoad, ObjectMaskOp, ObjectTrackStatus } from "@/engine/object-mask";
+
+const MODES = [
+ { value: "points", label: "Point", icon: "object-mask.point", shortcut: "P" },
+ { value: "brush", label: "Brush", icon: "mask-small", shortcut: "B" },
+] as const satisfies readonly { value: ObjectMaskMode; label: string; icon: string; shortcut: string }[];
+
+/** The popover's segments: labels only, the icons are the trigger's. */
+const MODE_TABS = MODES.map(({ value, label }) => ({ value, label }));
+
+const OPS = [
+ { value: "add", label: "Add", icon: "object-mask.add", hint: "Add to object" },
+ { value: "subtract", label: "Subtract", icon: "object-mask.subtract", hint: "Subtract from object" },
+] as const satisfies readonly { value: ObjectMaskOp; label: string; icon: string; hint: string }[];
+
+/** What each model is picked for: tiny at half resolution, the others at the full 1024. */
+const MODEL_NOTES: Record<Sam2ModelId, string> = {
+ tiny: "Fastest. Masks at half resolution, for large and clear objects.",
+ small: "Finer edges, at full resolution.",
+ "base-plus": "Holds on to small and thin objects.",
+ large: "Most accurate, and the slowest.",
+};
+
+/** The brush's size on its slider: its diameter, in percent of the frame's height. */
+const BRUSH_SIZE = { min: 1, max: 30 };
+
+/** The menus the bar opens: headed lists, laid out like the toolbar's own. */
+const MENU_CLASS = "rounded-xl border-border bg-background px-2 pt-0 pb-2 gap-0 shadow-[0px_12px_24px_0px_rgba(0,0,0,0.24)]";
+
+type SessionState = {
+ status: ObjectTrackStatus | null;
+ completed: number;
+ total: number;
+};
+
+/**
+ * The object mask tool's action bar, over the canvas while the tool is up:
+ * the model, how clicks prompt (points or the brush, adding to the object or
+ * subtracting from it), and the way out — Cancel drops the prompt, Confirm
+ * tracks the picked object through its clip and writes the mask into the
+ * document.
+ */
+export function ObjectMaskBar() {
+ const world = useWorld();
+
+ // The model is fetched as the tool opens, so it is ready by the first click.
+ onMount(preloadObjectMaskModel);
+
+ const state = useDerived<SessionState | null>(() => {
+ const track = getObjectTrack();
+ if (!track) return null;
+ return { status: track.status, completed: track.completed, total: track.masks.length };
+ }, sameState);
+
+ const busy = () => {
+ const status = state()?.status;
+ return status === "loading" || status === "segmenting" || status === "tracking" || status === "saving";
+ };
+
+ const confirmLabel = () => {
+ const session = state();
+ switch (session?.status) {
+ case "tracking":
+ return `Tracking ${Math.floor((session.total === 0 ? 0 : session.completed / session.total) * 100)}%`;
+ case "saving":
+ return "Saving...";
+ default:
+ return "Confirm";
+ }
+ };
+
+ // Where the mask goes: under the effect the tool was started for, else
+ // under an opacity effect on the clip — the cut-out.
+ const destination = () => {
+ const effect = targetEffect();
+ if (!effect?.isAlive()) return "Track the object through its clip and cut it out";
+ const clip = getParentNode(effect);
+ const label = effectOption(effect.get(Effect)?.type).label;
+ const name = clip?.get(Name)?.value;
+ return `Track the object and mask ${label}${name ? ` on ${name}` : ""}`;
+ };
+
+ return (
+ <div class="absolute bottom-16 left-1/2 -translate-x-1/2 z-10 rounded-xl px-1.5 py-1 bg-background border border-border flex gap-1 items-center">
+ <ModelMenu disabled={busy()} />
+ <Separator orientation="vertical" class="min-h-5" />
+ <ModePopover />
+ <OpMenu />
+ <Separator orientation="vertical" class="min-h-5" />
+ <Tooltip>
+ <TooltipTrigger as={Button} variant="ghost" class="text-muted-foreground" onClick={() => cancelObjectMask(world)}>
+ Cancel
+ </TooltipTrigger>
+ <TooltipContent shortcut="Esc">Cancel</TooltipContent>
+ </Tooltip>
+ <Tooltip>
+ <TooltipTrigger as={Button} disabled={state()?.status !== "seeded"} onClick={() => trackObjectMask(world)}>
+ {confirmLabel()}
+ </TooltipTrigger>
+ <TooltipContent shortcut="⌘↵">{destination()}</TooltipContent>
+ </Tooltip>
+ </div>
+ );
+}
+
+/** The model the tool segments with; each one's tooltip says what it is for and what it costs to fetch. */
+function ModelMenu(props: { disabled: boolean }) {
+ // The chosen model's load, while it is under way or failed.
+ const load = () => {
+ const current = objectMaskModelLoad();
+ return current && current.id === objectMaskModel() && current.phase !== "ready" ? current : null;
+ };
+ // A restore holds the model too: switching would take it away mid-way.
+ const restoring = useDerived(() => getMaskRestore() !== null);
+
+ // Which models are on this machine already, looked at again as loads finish.
+ const [cached] = createResource(
+ () => objectMaskModelLoad()?.phase ?? "idle",
+ async () => (await import("@diffusionstudio/sam2")).cachedSam2Models(),
+ );
+ const sizeNote = (model: Sam2Model) =>
+ cached()?.has(model.id) ? "Downloaded" : `${formatSize(downloadSize(model))} download`;
+
+ return (
+ <DropdownMenu placement="top-start" gutter={8}>
+ <Tooltip>
+ <TooltipTrigger<typeof DropdownMenuTrigger>
+ as={(triggerProps: object) => (
+ <DropdownMenuTrigger<typeof Button>
+ {...triggerProps}
+ disabled={props.disabled || restoring()}
+ as={(buttonProps) => (
+ <Button
+ {...buttonProps}
+ variant="ghost"
+ class={cx("gap-1", load()?.phase === "error" ? "gap-0 pl-0.5 text-destructive" : "text-muted-foreground")}
+ >
+ <Show when={load()?.phase === "error"}>
+ <Icon name="alert-warning" />
+ </Show>
+ {sam2Model(objectMaskModel()).label}
+ <Show when={load()?.phase === "download" || load()?.phase === "compile"}>
+ <span class="text-xxs font-normal tabular-nums">{loadProgress(load()!)}</span>
+ </Show>
+ </Button>
+ )}
+ />
+ )}
+ />
+ <TooltipContent>{load() ? describeModelLoad(load()!) : "Segmentation model"}</TooltipContent>
+ </Tooltip>
+ <DropdownMenuPortal>
+ <DropdownMenuContent class={cx(MENU_CLASS, "min-w-40")} onEscapeKeyDown={keepEscape}>
+ <MenuHeader>Model</MenuHeader>
+ <div class="flex flex-col gap-1 py-0.5">
+ <For each={SAM2_MODELS}>
+ {(model) => (
+ <Tooltip placement="right" gutter={12}>
+ <TooltipTrigger<typeof DropdownMenuItem>
+ as={(itemProps: object) => (
+ <CheckedItem
+ {...itemProps}
+ checked={model.id === objectMaskModel()}
+ onSelect={() => pickObjectMaskModel(model.id)}
+ >
+ {model.label}
+ </CheckedItem>
+ )}
+ />
+ <TooltipContent class="max-w-56 flex-col items-start gap-0.5">
+ <span>{MODEL_NOTES[model.id]}</span>
+ <span class="text-muted-foreground">{sizeNote(model)}</span>
+ </TooltipContent>
+ </Tooltip>
+ )}
+ </For>
+ </div>
+ </DropdownMenuContent>
+ </DropdownMenuPortal>
+ </DropdownMenu>
+ );
+}
+
+/** Points or the brush, and the brush's size. */
+function ModePopover() {
+ const mode = () => MODES.find((entry) => entry.value === objectMaskMode()) ?? MODES[0];
+
+ return (
+ <Popover placement="top-start" gutter={8}>
+ <Tooltip>
+ <TooltipTrigger<typeof PopoverTrigger>
+ as={(triggerProps: object) => (
+ <PopoverTrigger<typeof Button>
+ {...triggerProps}
+ as={(buttonProps) => (
+ <Button {...buttonProps} variant="ghost" class="gap-0 pl-0.5 text-muted-foreground">
+ <Icon name={mode().icon} />
+ {mode().label}
+ </Button>
+ )}
+ />
+ )}
+ />
+ <TooltipContent shortcut={mode().shortcut}>{mode().label}</TooltipContent>
+ </Tooltip>
+ <PopoverPortal>
+ <PopoverContent
+ class={cx(MENU_CLASS, "w-44")}
+ onEscapeKeyDown={keepEscape}
+ onOpenAutoFocus={(event) => event.preventDefault()}
+ >
+ <MenuHeader>Tool</MenuHeader>
+ <SegmentedIconTabs value={objectMaskMode} onChange={setObjectMaskMode} items={MODE_TABS} class="my-0.5" />
+ <Show when={objectMaskMode() === "brush"}>
+ <div class="pt-2 pb-0.5">
+ <SliderInput
+ value={Math.round(brushRadius() * 200)}
+ min={BRUSH_SIZE.min}
+ max={BRUSH_SIZE.max}
+ onChange={(value) => setBrushRadius(Math.min(BRUSH_SIZE.max, Math.max(BRUSH_SIZE.min, value)) / 200)}
+ format={(value) => `${value}%`}
+ />
+ </div>
+ </Show>
+ </PopoverContent>
+ </PopoverPortal>
+ </Popover>
+ );
+}
+
+/** Whether clicks add to the object or subtract from it; the button shows the op alt swaps in while held. */
+function OpMenu() {
+ const world = useWorld();
+ const held = useDerived(() => heldObjectMaskOp(world));
+ const op = () => OPS.find((entry) => entry.value === held()) ?? OPS[0];
+
+ return (
+ <DropdownMenu placement="top-start" gutter={8}>
+ <Tooltip>
+ <TooltipTrigger<typeof DropdownMenuTrigger>
+ as={(triggerProps: object) => (
+ <DropdownMenuTrigger<typeof Button>
+ {...triggerProps}
+ as={(buttonProps) => (
+ <Button {...buttonProps} variant="ghost" class="gap-0 pl-0.5 text-muted-foreground">
+ <Icon name={op().icon} />
+ {op().label}
+ </Button>
+ )}
+ />
+ )}
+ />
+ <TooltipContent shortcut="Hold ⌥ to swap">{op().hint}</TooltipContent>
+ </Tooltip>
+ <DropdownMenuPortal>
+ <DropdownMenuContent class={cx(MENU_CLASS, "min-w-40")} onEscapeKeyDown={keepEscape}>
+ <MenuHeader>Mode</MenuHeader>
+ <div class="flex flex-col gap-1 py-0.5">
+ <For each={OPS}>
+ {(entry) => (
+ <CheckedItem checked={entry.value === objectMaskOp()} onSelect={() => setObjectMaskOp(entry.value)}>
+ {entry.label}
+ </CheckedItem>
+ )}
+ </For>
+ </div>
+ </DropdownMenuContent>
+ </DropdownMenuPortal>
+ </DropdownMenu>
+ );
+}
+
+function MenuHeader(props: { children: JSX.Element }) {
+ return <div class="flex h-8 items-center px-1 text-xs text-muted-foreground">{props.children}</div>;
+}
+
+type CheckedItemProps = {
+ checked: boolean;
+ onSelect(): void;
+ children: JSX.Element;
+};
+
+/** A menu row that is the current pick: raised, with a check at its end. */
+function CheckedItem(props: CheckedItemProps) {
+ const [local, rest] = splitProps(props, ["checked", "children"]);
+
+ return (
+ <DropdownMenuItem
+ {...rest}
+ tone="neutral"
+ class={cx("pr-0", local.checked ? "bg-accent text-foreground" : "text-muted-foreground")}
+ >
+ <span class="min-w-0 flex-1 truncate">{local.children}</span>
+ <span class="flex size-7 items-center justify-center">
+ <Show when={local.checked}>
+ <Icon name="confirm-check" class="text-foreground" />
+ </Show>
+ </span>
+ </DropdownMenuItem>
+ );
+}
+
+/** Esc closes an open menu; it goes no further, where it would put the tool down. */
+function keepEscape(event: KeyboardEvent) {
+ event.stopPropagation();
+}
+
+function loadProgress(load: ObjectMaskModelLoad): string {
+ if (load.phase === "compile" || load.progress === null) return "...";
+ return `${Math.floor(load.progress * 100)}%`;
+}
+
+function describeModelLoad(load: ObjectMaskModelLoad): string {
+ const label = sam2Model(load.id).label;
+ switch (load.phase) {
+ case "download":
+ return load.progress === null ? `Downloading ${label}...` : `Downloading ${label}, ${Math.floor(load.progress * 100)}%`;
+ case "compile":
+ return `Preparing ${label}...`;
+ case "error":
+ return `${load.error ?? `${label} could not be loaded`}. Pick it again to retry.`;
+ default:
+ return "";
+ }
+}
+
+function formatSize(bytes: number): string {
+ return `${Math.round(bytes / 1e6)} MB`;
+}
+
+function sameState(a: SessionState | null, b: SessionState | null): boolean {
+ if (a === null || b === null) return a === b;
+ return a.status === b.status && a.completed === b.completed && a.total === b.total;
+}
diff --git a/apps/web/src/components/canvas/toolbar.tsx b/apps/web/src/components/canvas/toolbar.tsx
index 793f45b6..4fa77453 100644
--- a/apps/web/src/components/canvas/toolbar.tsx
+++ b/apps/web/src/components/canvas/toolbar.tsx
@@ -16,6 +16,7 @@ import {
} from "@/components/ui/dropdown-menu";
import { PromptInput } from "../genai/prompt-input";
import { ActionBar } from "../genai/action-bar";
+import { ObjectMaskBar } from "./object-mask-bar";
import { For, Show, createMemo } from "solid-js";
import { Tool, ToolType } from "@diffusionstudio/runtime";
import { useWorld } from "@diffusionstudio/koota-solid";
@@ -44,7 +45,12 @@ export function Toolbar() {
<PromptInput initialConfig={promptInputConfig()} />
</Show>
<Show when={!promptInputOpen()}>
- <ActionBar openPromptInput={openPromptInput} />
+ <Show
+ when={selectedTool() === ToolType.OBJECT_MASK}
+ fallback={<ActionBar openPromptInput={openPromptInput} />}
+ >
+ <ObjectMaskBar />
+ </Show>
</Show>
<div class="absolute bottom-4 left-1/2 -translate-x-1/2 rounded-xl p-1.5 bg-background border border-border-strong flex gap-2 items-center z-10">
<div class="flex gap-1">
@@ -87,8 +93,8 @@ export function Toolbar() {
<Icon name="confirm-check" class="text-foreground" />
</div>
<Icon name={cursor.menuIcon} class="text-foreground" />
- <span class="min-w-12 mx-1">{cursor.label}</span>
- <DropdownMenuShortcut>{cursor.shortcut}</DropdownMenuShortcut>
+ <span class="mx-1 flex-1 whitespace-nowrap">{cursor.label}</span>
+ <DropdownMenuShortcut class="pl-6">{cursor.shortcut}</DropdownMenuShortcut>
</DropdownMenuItem>
)}
</For>
diff --git a/apps/web/src/components/sidebar-right/inspector/inspector.tsx b/apps/web/src/components/sidebar-right/inspector/inspector.tsx
index 2084c7b3..6b74470f 100644
--- a/apps/web/src/components/sidebar-right/inspector/inspector.tsx
+++ b/apps/web/src/components/sidebar-right/inspector/inspector.tsx
@@ -22,7 +22,6 @@ import { InspectorHeader } from "./inspector-header";
import { BackgroundSettings } from "./background";
import { VariablesSettings } from "./variables";
import { SceneTemplatePanel } from "./scene-template";
-import { ObjectMaskPanel } from "./object-mask";
import { AssetInfoPanel } from "./asset-info";
import { TimeSettings } from "./time";
import { AppearanceSettings } from "./appearance";
@@ -48,7 +47,6 @@ import type { Entity } from "koota";
export type SelectionTarget =
| "scene-tool"
- | "mask-tool"
| "keyframe"
| "asset"
| "scene"
@@ -90,7 +88,6 @@ export function Inspector() {
const selectionTarget = createMemo<SelectionTarget>(() => {
if (tool() === ToolType.SCENE) return "scene-tool";
- if (tool() === ToolType.OBJECT_MASK) return "mask-tool";
if (keyframes().length > 0) return "keyframe";
if (asset() || partial()) return "asset";
const entity = first();
@@ -116,10 +113,6 @@ export function Inspector() {
<SceneTemplatePanel />
</Show>
- <Show when={includesTarget("mask-tool")}>
- <ObjectMaskPanel />
- </Show>
-
<Show when={nodes().length > 1}>
<Alignment />
</Show>
diff --git a/apps/web/src/components/sidebar-right/inspector/object-mask.tsx b/apps/web/src/components/sidebar-right/inspector/object-mask.tsx
index 30c0c549..b87856f2 100644
--- a/apps/web/src/components/sidebar-right/inspector/object-mask.tsx
+++ b/apps/web/src/components/sidebar-right/inspector/object-mask.tsx
@@ -2,8 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Show, createResource, onMount } from "solid-js";
-import { SAM2_MODELS, downloadSize, sam2Model } from "@diffusionstudio/sam2/models";
+import { Show } from "solid-js";
import { DEFAULT_MASK_SMOOTHING, assetName } from "@diffusionstudio/assets";
import { Button } from "@/components/ui/button";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
@@ -17,250 +16,24 @@ import {
import { Icon } from "@/components/ui/icon";
import { IncrementDecrementControl } from "@/components/ui/increment-decrement-control";
import { Keyframe } from "@/components/ui/keyframe";
-import { PanelSection } from "@/components/ui/panel-section";
-import { Progress } from "@/components/ui/progress";
-import { SegmentedIconTabs } from "@/components/ui/segmented-icon-tabs";
import { Select, SelectContent, SelectItem, SelectPortal, SelectTrigger, SelectValue } from "@/components/ui/select";
import { SliderInput } from "@/components/ui/slider-input";
import { Switch, SwitchControl, SwitchInput, SwitchThumb } from "@/components/ui/switch";
import { ControlledTextField } from "@/components/ui/text-field";
import { useHas, useTrait, useWorld } from "@diffusionstudio/koota-solid";
-import {
- AssetId, Computed, Effect, Hidden, Library, Mask, Name, VideoDecoderHandle, getParentNode,
-} from "@diffusionstudio/runtime";
+import { AssetId, Computed, Hidden, Library, Mask, VideoDecoderHandle, getParentNode } from "@diffusionstudio/runtime";
import { useDerived, useEditor } from "@/engine/hooks";
import { syncKeyframe } from "@/engine/keyframes";
import {
- brushRadius,
canRestoreObjectMask,
- clearObjectTrack,
- getMaskRestore,
getMaskRestoreOf,
- getObjectTrack,
- objectMaskMode,
- objectMaskModel,
- objectMaskModelLoad,
- pickObjectMaskModel,
- preloadObjectMaskModel,
restoreObjectMask,
- setBrushRadius,
- setObjectMaskMode,
- targetEffect,
- trackObjectMask,
- undoMaskStroke,
useObjectMasks,
} from "@/engine/object-mask";
-import { effectOption } from "./effect-types";
import type { Entity, World } from "koota";
import type { MaskAsset } from "@diffusionstudio/assets";
-import type { Sam2Model, Sam2ModelId } from "@diffusionstudio/sam2/models";
-import type {
- MaskRestore,
- ObjectMaskMode,
- ObjectMaskModelLoad,
- ObjectMaskSource,
- ObjectTrackStatus,
-} from "@/engine/object-mask";
-
-const MODE_TABS = [
- { value: "points", label: "Points" },
- { value: "brush", label: "Brush" },
-] as const satisfies readonly { value: ObjectMaskMode; label: string }[];
-
-/** What each model is picked for: tiny at half resolution, the others at the full 1024. */
-const MODEL_NOTES: Record<Sam2ModelId, string> = {
- tiny: "Fastest",
- small: "Finer edges",
- "base-plus": "Small objects",
- large: "Best, slowest",
-};
-
-/** The brush's size on its slider: its diameter, in percent of the frame's height. */
-const BRUSH_SIZE = { min: 1, max: 30 };
-
-type SessionState = {
- status: ObjectTrackStatus | null;
- completed: number;
- total: number;
- error: string | null;
- points: number;
- strokes: number;
-};
-
-/**
- * The object mask tool's panel, the sidebar while the tool is up: how to
- * prompt (hover a video for the segment under the pointer, click to pick),
- * where the session stands, and the button that tracks the picked object
- * through its clip and writes the mask into the document — an opacity
- * effect with a `<mask>` on the clip, whose settings then open from the
- * effect's inspector.
- */
-export function ObjectMaskPanel() {
- const world = useWorld();
-
- const state = useDerived<SessionState | null>(() => {
- const track = getObjectTrack();
- if (!track) return null;
- return {
- status: track.status,
- completed: track.completed,
- total: track.masks.length,
- error: track.error,
- points: track.points.length,
- strokes: track.strokes.length,
- };
- }, sameState);
-
- const busy = () => {
- const status = state()?.status;
- return status === "loading" || status === "segmenting" || status === "tracking" || status === "saving";
- };
-
- // Where the mask goes: under the effect the tool was started for, else
- // under an opacity effect on the clip — the cut-out.
- const destination = () => {
- const effect = targetEffect();
- if (!effect?.isAlive()) return "The tracked mask goes under an Opacity effect on the clip: the cut-out.";
- const clip = getParentNode(effect);
- const label = effectOption(effect.get(Effect)?.type).label;
- const name = clip?.get(Name)?.value;
- return `The tracked mask goes under ${label}${name ? ` on ${name}` : ""}.`;
- };
-
- // The model is fetched as the tool opens, so it is ready by the first click.
- onMount(preloadObjectMaskModel);
-
- // The chosen model's load, while it is under way or failed; a restore's other model is not shown.
- const modelLoad = () => {
- const load = objectMaskModelLoad();
- return load && load.id === objectMaskModel() && load.phase !== "ready" ? load : null;
- };
- // A restore holds the model too: switching would take it away mid-way.
- const restoring = useDerived(() => getMaskRestore() !== null);
-
- // Which models are on this machine already, looked at again as loads finish.
- const [cached] = createResource(
- () => objectMaskModelLoad()?.phase ?? "idle",
- async () => (await import("@diffusionstudio/sam2")).cachedSam2Models(),
- );
- const modelTag = (model: Sam2Model) =>
- cached()?.has(model.id) ? MODEL_NOTES[model.id] : `${MODEL_NOTES[model.id]} · ${formatSize(downloadSize(model))}`;
-
- return (
- <PanelSection title="Object Mask">
- <ControlRow label="Model">
- <Select<Sam2Model>
- value={sam2Model(objectMaskModel())}
- options={[...SAM2_MODELS]}
- optionValue="id"
- optionTextValue="label"
- disabled={busy() || restoring()}
- onChange={(model) => model && pickObjectMaskModel(model.id)}
- itemComponent={(itemProps) => (
- <SelectItem item={itemProps.item}>
- <div class="flex min-w-0 flex-1 items-center">
- <span>{itemProps.item.rawValue.label}</span>
- <span class="ml-auto pl-3 text-xxs text-muted-foreground group-[[data-highlighted]]:text-[inherit]">
- {modelTag(itemProps.item.rawValue)}
- </span>
- </div>
- </SelectItem>
- )}
- >
- <SelectTrigger>
- <SelectValue<Sam2Model>>
- {(select) => (
- <>
- <span>{select.selectedOption()?.label}</span>
- <span class="ml-auto text-xxs text-muted-foreground">
- {select.selectedOption() && modelTag(select.selectedOption())}
- </span>
- </>
- )}
- </SelectValue>
- </SelectTrigger>
- <SelectPortal>
- <SelectContent />
- </SelectPortal>
- </Select>
- </ControlRow>
- <Show when={modelLoad()}>
- {(load) => (
- <div class="flex flex-col gap-2">
- <span class="text-xs" classList={{ "text-destructive": load().phase === "error" }}>
- {describeModelLoad(load())}
- </span>
- <Show when={load().phase === "download"}>
- <Progress value={load().progress ?? 0} minValue={0} maxValue={1} />
- </Show>
- <Show when={load().phase === "error"}>
- <div>
- <Button size="small" variant="ghost" onClick={preloadObjectMaskModel}>
- Retry
- </Button>
- </div>
- </Show>
- </div>
- )}
- </Show>
- <SegmentedIconTabs value={objectMaskMode} onChange={setObjectMaskMode} items={MODE_TABS} />
- <Show
- when={objectMaskMode() === "brush"}
- fallback={
- <p class="text-xs text-muted-foreground">
- Hover a video clip to see what a click picks. Click to mask the object on this frame; more clicks refine
- it, alt-click marks background. Then track it through the clip.
- </p>
- }
- >
- <p class="text-xs text-muted-foreground">
- Paint over the mask to add what it missed; alt-drag erases what it should not have. Click the object with
- Points first: the brush corrects the mask, and tracking follows the corrected shape.
- </p>
- <ControlRow label="Size">
- <SliderInput
- value={Math.round(brushRadius() * 200)}
- min={BRUSH_SIZE.min}
- max={BRUSH_SIZE.max}
- onChange={(value) => setBrushRadius(Math.min(BRUSH_SIZE.max, Math.max(BRUSH_SIZE.min, value)) / 200)}
- format={(value) => `${value}%`}
- />
- </ControlRow>
- </Show>
- <p class="text-xs text-muted-foreground">{destination()}</p>
-
- <Show when={state()}>
- {(session) => (
- <div class="flex flex-col gap-2">
- <span class="text-xs" classList={{ "text-destructive": session().status === "error" }}>
- {describe(session())}
- </span>
- <Show when={session().status === "tracking"}>
- <Progress value={progress(session())} minValue={0} maxValue={1} />
- </Show>
- </div>
- )}
- </Show>
-
- <div class="flex items-center gap-2">
- <Button size="small" disabled={state()?.status !== "seeded"} onClick={() => trackObjectMask(world)}>
- Track mask
- </Button>
- <Show when={state()?.status === "seeded" && state()!.strokes > 0}>
- <Button size="small" variant="ghost" onClick={() => undoMaskStroke(world)}>
- Undo stroke
- </Button>
- </Show>
- <Show when={state()}>
- <Button size="small" variant="ghost" onClick={clearObjectTrack}>
- {busy() ? "Cancel" : "Clear"}
- </Button>
- </Show>
- </div>
- </PanelSection>
- );
-}
+import type { MaskRestore, ObjectMaskSource } from "@/engine/object-mask";
type ObjectMaskInspectorProps = {
/** The `<mask>`; its `src` is the tracked picture. */
@@ -481,49 +254,6 @@ export function objectMaskName(world: World, mask: Entity): string {
return asset ? assetName(asset).replace(/\.[^.]+$/, "") : "Mask";
}
-function describe(state: SessionState): string {
- const strokes = state.strokes === 0 ? "" : ` and ${state.strokes} ${state.strokes === 1 ? "stroke" : "strokes"}`;
- const points = `${state.points} ${state.points === 1 ? "point" : "points"}${strokes}`;
- switch (state.status) {
- case "loading":
- return "Waiting for the model...";
- case "segmenting":
- return `Masking from ${points}...`;
- case "seeded":
- return `Masked this frame from ${points}. Track it to mask the whole clip.`;
- case "tracking":
- return `Tracking, ${state.completed} of ${state.total} frames`;
- case "saving":
- return "Saving the mask...";
- case "error":
- return state.error ?? "Tracking failed";
- default:
- return "";
- }
-}
-
-function progress(state: SessionState): number {
- return state.total === 0 ? 0 : state.completed / state.total;
-}
-
-function describeModelLoad(load: ObjectMaskModelLoad): string {
- const label = sam2Model(load.id).label;
- switch (load.phase) {
- case "download":
- return load.progress === null ? `Downloading ${label}...` : `Downloading ${label}, ${Math.floor(load.progress * 100)}%`;
- case "compile":
- return `Preparing ${label}...`;
- case "error":
- return load.error ?? `${label} could not be loaded`;
- default:
- return "";
- }
-}
-
-function formatSize(bytes: number): string {
- return `${Math.round(bytes / 1e6)} MB`;
-}
-
function describeRestore(restore: MaskRestore): string {
switch (restore.status) {
case "loading":
@@ -537,7 +267,3 @@ function describeRestore(restore: MaskRestore): string {
}
}
-function sameState(a: SessionState | null, b: SessionState | null): boolean {
- if (a === null || b === null) return a === b;
- return (Object.keys(a) as (keyof SessionState)[]).every((key) => a[key] === b[key]);
-}
diff --git a/apps/web/src/engine/input/shortcuts.ts b/apps/web/src/engine/input/shortcuts.ts
index 21ac53b8..fe03e4d7 100644
--- a/apps/web/src/engine/input/shortcuts.ts
+++ b/apps/web/src/engine/input/shortcuts.ts
@@ -37,6 +37,7 @@ import { zoomBy, zoomTo, zoomToFit, zoomToSelection } from '../camera';
import { getDocumentEditor } from '../editor';
import { groupSelection, ungroupSelection, unwrapSequenceSelection, wrapSelectionInScene, wrapSelectionInSequence } from '../group';
import { getEditHistory } from '../history';
+import { cancelObjectMask, getObjectTrack, setObjectMaskMode, trackObjectMask, undoMaskStroke } from '../object-mask';
import { splitAtPlayhead } from '../split';
import { Keys, MODIFIER_KEYS, Pointer } from '../traits';
import { editTransform } from './interactions';
@@ -48,6 +49,7 @@ import type { Entity, World } from 'koota';
type Shortcut = {
keys: string[];
action: (world: World) => void;
+ active?: (world: World) => boolean;
}
/** The node kinds a shortcut selects, hides or seeks around. */
@@ -442,6 +444,14 @@ function deselect(world: World): void {
if (tool !== ToolType.MOVE && tool !== ToolType.HAND) selectTool(ToolType.MOVE)(world);
}
+const objectMaskTool = (world: World): boolean => world.get(Tool)?.value === ToolType.OBJECT_MASK;
+
+/** A brush stroke to take back: ⌘Z undoes strokes before it undoes edits. */
+const maskStrokeToUndo = (world: World): boolean => {
+ const track = getObjectTrack();
+ return objectMaskTool(world) && track?.status === 'seeded' && track.strokes.length > 0;
+};
+
const PRESSED_SHORTCUTS: readonly Shortcut[] = [
{ keys: ['z', 'mod', '!shift'], action: undoEdit },
{ keys: ['z', 'mod', 'shift'], action: redoEdit },
@@ -497,6 +507,12 @@ const PRESSED_SHORTCUTS: readonly Shortcut[] = [
{ keys: ['arrowup', 'shift'], action: nudge(0, -NUDGE_FAST) },
{ keys: ['arrowdown', 'shift'], action: nudge(0, NUDGE_FAST) },
{ keys: [' '], action: onSpacePressed },
+ // Object mask tool shortcuts
+ { keys: ['p', '!mod'], action: () => setObjectMaskMode('points'), active: objectMaskTool },
+ { keys: ['b', '!mod'], action: () => setObjectMaskMode('brush'), active: objectMaskTool },
+ { keys: ['escape'], action: cancelObjectMask, active: objectMaskTool },
+ { keys: ['enter', 'mod', '!shift', '!alt'], action: trackObjectMask, active: objectMaskTool },
+ { keys: ['z', 'mod', '!shift'], action: undoMaskStroke, active: maskStrokeToUndo },
];
const LIFTED_SHORTCUTS: readonly Shortcut[] = [
@@ -515,7 +531,7 @@ const LIFTED_SHORTCUTS: readonly Shortcut[] = [
* the key-up of a key released while ⌘ is down, so a fresh ⌘ press must not
* complete a shortcut on its own.
*/
-function matches(shortcut: Shortcut, moved: Set<string>, held: Set<string>): boolean {
+function matches(world: World, shortcut: Shortcut, moved: Set<string>, held: Set<string>): boolean {
let triggered = false;
for (const key of shortcut.keys) {
@@ -528,7 +544,7 @@ function matches(shortcut: Shortcut, moved: Set<string>, held: Set<string>): boo
}
}
- return triggered;
+ return triggered && (shortcut.active?.(world) ?? true);
}
/** On a frame with a fresh press or release, runs the shortcut it spells. */
@@ -537,11 +553,11 @@ export function shortcutSystem(world: World): void {
if (!keys) return;
if (keys.pressed.size) {
- PRESSED_SHORTCUTS.find(shortcut => matches(shortcut, keys.pressed, keys.held))?.action(world);
+ PRESSED_SHORTCUTS.find(shortcut => matches(world, shortcut, keys.pressed, keys.held))?.action(world);
}
if (keys.lifted.size) {
- LIFTED_SHORTCUTS.find(shortcut => matches(shortcut, keys.lifted, keys.held))?.action(world);
+ LIFTED_SHORTCUTS.find(shortcut => matches(world, shortcut, keys.lifted, keys.held))?.action(world);
}
updateSpaceHold(world, keys.held);
diff --git a/apps/web/src/engine/object-mask/interaction.ts b/apps/web/src/engine/object-mask/interaction.ts
index 7b720e0e..b8a793aa 100644
--- a/apps/web/src/engine/object-mask/interaction.ts
+++ b/apps/web/src/engine/object-mask/interaction.ts
@@ -7,11 +7,12 @@ import { RenderSurface } from '@diffusionstudio/runtime';
import { Keys, Pointer } from '../traits';
import { beginMaskStroke, endMaskStroke, extendMaskStroke } from './brush';
import { getVideoRect, pointOnVideo } from './media';
-import { objectMaskMode } from './store';
+import { objectMaskMode, objectMaskOp } from './store';
import { promptObjectMask } from './tracking';
import type { World } from 'koota';
import type { DispatchedPointerEvent } from '@diffusionstudio/runtime';
+import type { ObjectMaskOp } from './store';
/** A pointer that traveled less than this, in CSS pixels, between press and release is a click. */
const CLICK_DISTANCE = 4;
@@ -19,7 +20,8 @@ const CLICK_DISTANCE = 4;
/**
* The handler for the region the HUD lays over the tool's clip. With points,
* a click prompts the object under the pointer; with the brush, a drag paints
- * it into the mask. With alt or shift held either marks background instead.
+ * it into the mask. Either adds to the object or subtracts from it, as the
+ * tool's op says; alt or shift held swaps the op.
*/
export function handleObjectMaskInteraction(world: World, event: DispatchedPointerEvent): void {
if (objectMaskMode() === 'brush') {
@@ -39,7 +41,7 @@ export function handleObjectMaskInteraction(world: World, event: DispatchedPoint
const point = pointOnVideo(rect, event.clientX, event.clientY);
if (!point) return;
- promptObjectMask(world, clip, { ...point, label: backgroundHeld(world) ? 0 : 1 });
+ promptObjectMask(world, clip, { ...point, label: heldObjectMaskLabel(world) });
}
/** A press starts a stroke on the video, every move until the release carries it on. */
@@ -47,7 +49,7 @@ function handleBrush(world: World, event: DispatchedPointerEvent): void {
switch (event.type) {
case 'dragstart':
if (event.target.kind !== 'hud' || !event.target.entity) return;
- beginMaskStroke(world, event.target.entity, event.clientX, event.clientY, backgroundHeld(world) ? 0 : 1);
+ beginMaskStroke(world, event.target.entity, event.clientX, event.clientY, heldObjectMaskLabel(world));
return;
case 'drag':
extendMaskStroke(world, event.clientX, event.clientY);
@@ -58,8 +60,16 @@ function handleBrush(world: World, event: DispatchedPointerEvent): void {
}
}
-/** Whether alt or shift is held: the gesture marks background rather than the object. */
-export function backgroundHeld(world: World): boolean {
+/** The op a gesture makes now: the tool's own, swapped while alt or shift is held. */
+export function heldObjectMaskOp(world: World): ObjectMaskOp {
const held = world.get(Keys)?.held;
- return !!held && (held.has('alt') || held.has('shift'));
+ const swapped = !!held && (held.has('alt') || held.has('shift'));
+ const op = objectMaskOp();
+ if (!swapped) return op;
+ return op === 'add' ? 'subtract' : 'add';
+}
+
+/** The prompt label a gesture carries now: 1 adds to the object, 0 subtracts from it. */
+export function heldObjectMaskLabel(world: World): 0 | 1 {
+ return heldObjectMaskOp(world) === 'add' ? 1 : 0;
}
diff --git a/apps/web/src/engine/object-mask/overlay.ts b/apps/web/src/engine/object-mask/overlay.ts
index 86b3e2cb..2066e477 100644
--- a/apps/web/src/engine/object-mask/overlay.ts
+++ b/apps/web/src/engine/object-mask/overlay.ts
@@ -9,7 +9,7 @@ import { geometryOf } from '@diffusionstudio/sam2/models';
import { Pointer } from '../traits';
import { endMaskStroke, paintableObjectTrack } from './brush';
import { maskFrame } from './frame';
-import { backgroundHeld, handleObjectMaskInteraction } from './interaction';
+import { handleObjectMaskInteraction, heldObjectMaskLabel } from './interaction';
import { currentSourceFrame, getVideoRect, pointOnVideo, videoPointAt, videoPointToDevice } from './media';
import {
brushRadius, clearTargetEffect, getObjectHover, getObjectMask, getObjectTrack, getTargetClip, objectMaskMode,
@@ -43,7 +43,7 @@ const outlines = new Map<Sam2Mask | MaskFrame, Path2D>();
/** Whether the tool was up last frame, so putting it down cleans up once. */
let active = false;
-let lastHover: { frame: number; x: number; y: number } | null = null;
+let lastHover: { frame: number; x: number; y: number; label: 0 | 1 } | null = null;
/**
* Everything the tool puts on the canvas while it is up: the session over its
@@ -114,9 +114,13 @@ function updateHover(world: World, target: { clip: Entity; rect: VideoRect } | n
}
const frame = currentSourceFrame(world, target.clip);
- if (lastHover && lastHover.frame === frame && Math.hypot(point.x - lastHover.x, point.y - lastHover.y) < HOVER_STEP) return;
- lastHover = { frame, x: point.x, y: point.y };
- hoverObjectMask(world, target.clip, { ...point, label: 1 });
+ const label = heldObjectMaskLabel(world);
+ if (
+ lastHover && lastHover.frame === frame && lastHover.label === label
+ && Math.hypot(point.x - lastHover.x, point.y - lastHover.y) < HOVER_STEP
+ ) return;
+ lastHover = { frame, x: point.x, y: point.y, label };
+ hoverObjectMask(world, target.clip, { ...point, label });
}
/**
@@ -207,7 +211,7 @@ function drawBrush(world: World, ctx: Ctx2D, target: { clip: Entity; rect: Video
ctx.strokeStyle = 'rgba(0, 0, 0, 0.4)';
ctx.stroke();
ctx.lineWidth = 1.5 * resolution;
- ctx.strokeStyle = backgroundHeld(world) ? BACKGROUND_POINT : '#FFFFFF';
+ ctx.strokeStyle = heldObjectMaskLabel(world) === 0 ? BACKGROUND_POINT : '#FFFFFF';
ctx.stroke();
ctx.restore();
}
diff --git a/apps/web/src/engine/object-mask/store.ts b/apps/web/src/engine/object-mask/store.ts
index 956e3400..fee14314 100644
--- a/apps/web/src/engine/object-mask/store.ts
+++ b/apps/web/src/engine/object-mask/store.ts
@@ -129,14 +129,22 @@ export function setObjectHover(next: ObjectHover | null): void {
/** Clicks prompt the model with points; the brush paints corrections over the mask it made. */
export type ObjectMaskMode = 'points' | 'brush';
+/**
+ * Whether a click or a stroke adds to the object (label 1) or subtracts from
+ * it (label 0). Holding alt swaps it for as long as the key is down (see
+ * `heldObjectMaskOp`).
+ */
+export type ObjectMaskOp = 'add' | 'subtract';
+
/** The brush's radius when the tool is first picked, in 0..1 of the frame's height. */
export const DEFAULT_BRUSH_RADIUS = 0.03;
const [objectMaskMode, setObjectMaskMode] = createSignal<ObjectMaskMode>('points');
+const [objectMaskOp, setObjectMaskOp] = createSignal<ObjectMaskOp>('add');
const [brushRadius, setBrushRadius] = createSignal(DEFAULT_BRUSH_RADIUS);
-/** The tool's mode and brush size, reactive; kept while the tool is put down. */
-export { objectMaskMode, setObjectMaskMode, brushRadius, setBrushRadius };
+/** The tool's mode, op and brush size, reactive; kept while the tool is put down. */
+export { objectMaskMode, setObjectMaskMode, objectMaskOp, setObjectMaskOp, brushRadius, setBrushRadius };
// ── The model ────────────────────────────────────────────────
diff --git a/apps/web/src/engine/object-mask/tracking.ts b/apps/web/src/engine/object-mask/tracking.ts
index b150d97f..77a7b68b 100644
--- a/apps/web/src/engine/object-mask/tracking.ts
+++ b/apps/web/src/engine/object-mask/tracking.ts
@@ -144,6 +144,19 @@ export function trackObjectMask(world: World): void {
});
}
+/**
+ * Puts the tool down without a mask: whatever the session was computing
+ * stops, its prompt is dropped, and the Move tool takes over.
+ */
+export function cancelObjectMask(world: World): void {
+ clearObjectTrack();
+ clearTargetEffect();
+
+ if (world.get(Tool)?.value === ToolType.OBJECT_MASK) {
+ world.set(Tool, { value: ToolType.MOVE });
+ }
+}
+
// ── Hover ────────────────────────────────────────────────────
/** The latest point the pointer asked about while a preview was running; served next. */
diff --git a/packages/sam2/src/constants.ts b/packages/sam2/src/constants.ts
index 9de98894..654918d0 100644
--- a/packages/sam2/src/constants.ts
+++ b/packages/sam2/src/constants.ts
@@ -28,7 +28,7 @@ export type Sam2Model = {
export const SAM2_MODELS: readonly Sam2Model[] = [
{
id: 'tiny',
- label: 'Tiny',
+ label: 'SAM 2.1 Tiny',
repo: 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/66673b5db39371b7dd7847f4d3bc0d4f4179b79e',
imageSize: 512,
files: {
@@ -42,7 +42,7 @@ export const SAM2_MODELS: readonly Sam2Model[] = [
},
{
id: 'small',
- label: 'Small',
+ label: 'SAM 2.1 Small',
repo: 'https://huggingface.co/diffusionstudio/sam2.1-small-video-onnx-fp16/resolve/927ecec6e6ae2727d0af7720eefdb07f2efe4689',
imageSize: 1024,
files: {
@@ -56,7 +56,7 @@ export const SAM2_MODELS: readonly Sam2Model[] = [
},
{
id: 'base-plus',
- label: 'Base+',
+ label: 'SAM 2.1 Base',
repo: 'https://huggingface.co/diffusionstudio/sam2.1-base-plus-video-onnx-fp16/resolve/386b83e09a774fe2782d86fd1b8cb5aaf718aa9d',
imageSize: 1024,
files: {
@@ -70,7 +70,7 @@ export const SAM2_MODELS: readonly Sam2Model[] = [
},
{
id: 'large',
- label: 'Large',
+ label: 'SAM 2.1 Large',
repo: 'https://huggingface.co/diffusionstudio/sam2.1-large-video-onnx-fp16/resolve/98dbd222d9b3f75d15fe63a5c2a22b0b0029eea1',
imageSize: 1024,
files: {