The VS Code of video editing, built for agents.
TypeScript
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { For, Show, createResource, createSignal, onMount, splitProps } from "solid-js";
import { Portal } from "solid-js/web";
import { SAM2_MODELS, downloadSize, sam2Model } from "@diffusionstudio/sam2/models";
import { useWorld } from "@diffusionstudio/koota-solid";
import { Button } from "@/components/ui/button";
import { Icon } from "@/components/ui/icon";
import { Separator } from "@/components/ui/separator";
import { SegmentedIconTabs } from "@/components/ui/segmented-icon-tabs";
import { SliderInput } from "@/components/ui/slider-input";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { Popover, PopoverContent, PopoverPortal, PopoverTrigger } from "@/components/ui/popover";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuPortal,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { cx } from "@/lib/cva";
import { ActionBar } from "./action-bar";
import { useDerived, useObjectMaskTool } from "@/engine/hooks";
import {
cancelObjectMask,
getMaskRestore,
getObjectTrack,
heldObjectMaskOp,
objectMaskModel,
objectMaskModelLoad,
pickObjectMaskModel,
preloadObjectMaskModel,
trackObjectMask,
} from "@/engine/object-mask";
import type { JSX } from "solid-js";
import type { Sam2Model, Sam2ModelId } from "@diffusionstudio/sam2/models";
import type { ObjectMaskMode, ObjectMaskModelLoad, ObjectMaskOp, ObjectTrackStatus } from "@/engine/object-mask";
const MODES = [
{ value: "points", label: "Point", icon: "object-mask.point", shortcut: "P" },
{ value: "brush", label: "Brush", icon: "mask-small", shortcut: "B" },
] as const satisfies readonly { value: ObjectMaskMode; label: string; icon: string; shortcut: string }[];
/** The popover's segments: labels only, the icons are the trigger's. */
const MODE_TABS = MODES.map(({ value, label }) => ({ value, label }));
const OPS = [
{ value: "add", label: "Add", icon: "object-mask.add", hint: "Add to object" },
{ value: "subtract", label: "Subtract", icon: "object-mask.subtract", hint: "Subtract from object" },
] as const satisfies readonly { value: ObjectMaskOp; label: string; icon: string; hint: string }[];
/** When to reach for each model: tiny runs at half resolution, the others at the full 1024. */
const MODEL_NOTES: Record<Sam2ModelId, string> = {
tiny: "Use for quick masks of large, clear subjects. Fastest, at half resolution.",
small: "Use when Tiny's edges are too rough.",
"base-plus": "Use for small or thin objects that Small loses.",
large: "Use for hard shots, when accuracy matters more than speed.",
};
/** The brush's size on its slider: its diameter, in percent of the frame's height. */
const BRUSH_SIZE = { min: 1, max: 30 };
/** The menus the bar opens: headed lists, laid out like the toolbar's own. */
const MENU_CLASS = "rounded-xl border-border bg-background px-2 pt-0 pb-2 gap-0 shadow-[0px_12px_24px_0px_rgba(0,0,0,0.24)]";
type SessionState = {
status: ObjectTrackStatus | null;
completed: number;
total: number;
};
/**
* The object mask tool's action bar, over the canvas while the tool is up:
* the model, how clicks prompt (points or the brush, adding to the object or
* subtracting from it), and the way out — Cancel drops the prompt, Confirm
* tracks the picked object through its clip and writes the mask into the
* document.
*/
export function ObjectMaskBar() {
const world = useWorld();
const tool = useObjectMaskTool();
// The model is fetched as the tool opens, so it is ready by the first click.
onMount(preloadObjectMaskModel);
const state = useDerived<SessionState | null>(() => {
const track = getObjectTrack();
if (!track) return null;
return { status: track.status, completed: track.completed, total: track.masks.length };
}, sameState);
const busy = () => {
const status = state()?.status;
return status === "loading" || status === "segmenting" || status === "tracking" || status === "saving";
};
const confirmLabel = () => {
const session = state();
switch (session?.status) {
case "tracking":
return (
<FixedWidth widest="Tracking 100%">
{`Tracking ${percent(session.total === 0 ? 0 : session.completed / session.total)}`}
</FixedWidth>
);
case "saving":
return "Saving...";
default:
return "Confirm";
}
};
return (
<ActionBar>
<ModelMenu disabled={busy()} />
<Separator orientation="vertical" class="min-h-5" />
<ModePopover />
<OpMenu disabled={tool.mode() === "brush"} />
<Separator orientation="vertical" class="min-h-5" />
<Tooltip>
<TooltipTrigger as={Button} variant="ghost" class="text-muted-foreground" onClick={() => cancelObjectMask(world)}>
Cancel
</TooltipTrigger>
<TooltipContent shortcut="Esc">Cancel</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger as={Button} disabled={state()?.status !== "seeded"} onClick={() => trackObjectMask(world)}>
{confirmLabel()}
</TooltipTrigger>
<TooltipContent shortcut="⌘↵"></TooltipContent>
</Tooltip>
</ActionBar>
);
}
/**
* The model the tool segments with. Hovering a model shows, to the left of
* its row, when to use it and what it costs to fetch.
*/
function ModelMenu(props: { disabled: boolean }) {
// The chosen model's load, while it is under way or failed.
const load = () => {
const current = objectMaskModelLoad();
return current && current.id === objectMaskModel() && current.phase !== "ready" ? current : null;
};
// A restore holds the model too: switching would take it away mid-way.
const restoring = useDerived(() => getMaskRestore() !== null);
// Which models are on this machine already, looked at again as loads finish.
const [cached] = createResource(
() => objectMaskModelLoad()?.phase ?? "idle",
async () => (await import("@diffusionstudio/sam2")).cachedSam2Models(),
);
// The row under the pointer (or keyboard), whose hint shows beside it.
const [hovered, setHovered] = createSignal<{ model: Sam2Model; row: HTMLElement } | null>(null);
const sizeNote = (model: Sam2Model) =>
`${formatSize(downloadSize(model))}${cached()?.has(model.id) ? ", downloaded" : " download"}`;
return (
<DropdownMenu placement="top-start" gutter={8} onOpenChange={(open) => !open && setHovered(null)}>
<Tooltip>
<TooltipTrigger<typeof DropdownMenuTrigger>
as={(triggerProps: object) => (
<DropdownMenuTrigger<typeof Button>
{...triggerProps}
disabled={props.disabled || restoring()}
as={(buttonProps) => (
<Button
{...buttonProps}
variant="ghost"
class={cx("gap-1", load()?.phase === "error" ? "gap-0 pl-0.5 text-destructive" : "text-muted-foreground")}
>
<Show when={load()?.phase === "error"}>
<Icon name="alert-warning" />
</Show>
<Show
when={load()?.phase === "download" || load()?.phase === "compile"}
fallback={sam2Model(objectMaskModel()).label}
>
<FixedWidth widest={<ModelProgress label={sam2Model(objectMaskModel()).label} percent="100%" />}>
<ModelProgress
label={sam2Model(objectMaskModel()).label}
percent={percent(load()!.phase === "download" ? load()!.progress : null)}
/>
</FixedWidth>
</Show>
</Button>
)}
/>
)}
/>
<TooltipContent>{load() ? describeModelLoad(load()!) : "Segmentation model"}</TooltipContent>
</Tooltip>
<DropdownMenuPortal>
<DropdownMenuContent class={cx(MENU_CLASS, "min-w-40")} onEscapeKeyDown={keepEscape}>
<MenuHeader>Model</MenuHeader>
<div class="flex flex-col gap-1 py-0.5">
<For each={SAM2_MODELS}>
{(model) => (
<CheckedItem
checked={model.id === objectMaskModel()}
onSelect={() => pickObjectMaskModel(model.id)}
onHover={(row) => setHovered(row ? { model, row } : null)}
>
{model.label}
</CheckedItem>
)}
</For>
</div>
</DropdownMenuContent>
</DropdownMenuPortal>
<Show when={hovered()}>
{(hover) => (
<Portal>
<ModelHint row={hover().row}>
<span>{MODEL_NOTES[hover().model.id]}</span>
<span class="text-muted-foreground">{sizeNote(hover().model)}</span>
</ModelHint>
</Portal>
)}
</Show>
</DropdownMenu>
);
}
/** A tooltip-like note 12px to the left of `row`, centered on it. */
function ModelHint(props: { row: HTMLElement; children: JSX.Element }) {
const rect = () => props.row.getBoundingClientRect();
return (
<div
role="tooltip"
class="pointer-events-none fixed z-[10001] flex max-w-56 -translate-x-full -translate-y-1/2 flex-col gap-0.5 rounded-md border border-border bg-background px-2 py-1 text-xs font-450 text-foreground shadow-md"
style={{ left: `${rect().left - 12}px`, top: `${rect().top + rect().height / 2}px` }}
>
{props.children}
</div>
);
}
/** Points or the brush, and the brush's size. */
function ModePopover() {
const tool = useObjectMaskTool();
const mode = () => MODES.find((entry) => entry.value === tool.mode()) ?? MODES[0];
return (
<Popover placement="top-start" gutter={8}>
<Tooltip>
<TooltipTrigger<typeof PopoverTrigger>
as={(triggerProps: object) => (
<PopoverTrigger<typeof Button>
{...triggerProps}
as={(buttonProps) => (
<Button {...buttonProps} variant="ghost" class="gap-0 pl-0.5 text-muted-foreground">
<Icon name={mode().icon} />
{mode().label}
</Button>
)}
/>
)}
/>
<TooltipContent shortcut={mode().shortcut}>{mode().label}</TooltipContent>
</Tooltip>
<PopoverPortal>
<PopoverContent
class={cx(MENU_CLASS, "w-44")}
onEscapeKeyDown={keepEscape}
onOpenAutoFocus={(event) => event.preventDefault()}
>
<MenuHeader>Tool</MenuHeader>
<SegmentedIconTabs value={tool.mode} onChange={(mode) => tool.set({ mode })} items={MODE_TABS} class="my-0.5" />
<Show when={mode().value === "brush"}>
<div class="pt-2 pb-0.5">
<SliderInput
value={Math.round(tool.brushRadius() * 200)}
min={BRUSH_SIZE.min}
max={BRUSH_SIZE.max}
onChange={(value) => tool.set({ brushRadius: Math.min(BRUSH_SIZE.max, Math.max(BRUSH_SIZE.min, value)) / 200 })}
format={(value) => `${value}%`}
/>
</div>
</Show>
</PopoverContent>
</PopoverPortal>
</Popover>
);
}
/** Whether clicks add to the object or subtract from it; the button shows the op alt swaps in while held. */
function OpMenu(props: { disabled: boolean }) {
const world = useWorld();
const tool = useObjectMaskTool();
const held = useDerived(() => heldObjectMaskOp(world));
const op = () => (props.disabled ? OPS[0] : OPS.find((entry) => entry.value === held()) ?? OPS[0]);
return (
<DropdownMenu placement="top-start" gutter={8}>
<Tooltip>
<TooltipTrigger<typeof DropdownMenuTrigger>
as={(triggerProps: object) => (
<DropdownMenuTrigger<typeof Button>
{...triggerProps}
disabled={props.disabled}
as={(buttonProps) => (
<Button {...buttonProps} variant="ghost" class="gap-0 pl-0.5 text-muted-foreground">
<Icon name={op().icon} />
{op().label}
</Button>
)}
/>
)}
/>
<TooltipContent shortcut="Hold ⌥ to swap">{op().hint}</TooltipContent>
</Tooltip>
<DropdownMenuPortal>
<DropdownMenuContent class={cx(MENU_CLASS, "min-w-40")} onEscapeKeyDown={keepEscape}>
<MenuHeader>Mode</MenuHeader>
<div class="flex flex-col gap-1 py-0.5">
<For each={OPS}>
{(entry) => (
<CheckedItem checked={entry.value === tool.op()} onSelect={() => tool.set({ op: entry.value })}>
{entry.label}
</CheckedItem>
)}
</For>
</div>
</DropdownMenuContent>
</DropdownMenuPortal>
</DropdownMenu>
);
}
function MenuHeader(props: { children: JSX.Element }) {
return <div class="flex h-8 items-center px-1 text-xs text-muted-foreground">{props.children}</div>;
}
type CheckedItemProps = {
checked: boolean;
onSelect(): void;
onHover?(row: HTMLElement | null): void;
children: JSX.Element;
};
/** A menu row that is the current pick: raised, with a check at its end. */
function CheckedItem(props: CheckedItemProps) {
const [local, rest] = splitProps(props, ["checked", "onHover", "children"]);
return (
<DropdownMenuItem
{...rest}
onPointerEnter={(event: PointerEvent) => local.onHover?.(event.currentTarget as HTMLElement)}
onPointerLeave={() => local.onHover?.(null)}
onFocus={(event: FocusEvent) => local.onHover?.(event.currentTarget as HTMLElement)}
onBlur={() => local.onHover?.(null)}
tone="neutral"
class={cx("pr-0", local.checked ? "bg-accent text-foreground" : "text-muted-foreground")}
>
<span class="min-w-0 flex-1 truncate">{local.children}</span>
<span class="flex size-7 items-center justify-center">
<Show when={local.checked}>
<Icon name="confirm-check" class="text-foreground" />
</Show>
</span>
</DropdownMenuItem>
);
}
/** Esc closes an open menu; it goes no further, where it would put the tool down. */
function keepEscape(event: KeyboardEvent) {
event.stopPropagation();
}
/**
* Content as wide as `widest` whatever it says, centered in that width: the
* button around it holds still as a number in it counts up.
*/
function FixedWidth(props: { widest: JSX.Element; children: JSX.Element }) {
return (
<span class="inline-grid tabular-nums">
<span class="invisible col-start-1 row-start-1 flex items-center justify-center gap-1" aria-hidden="true">
{props.widest}
</span>
<span class="col-start-1 row-start-1 flex items-center justify-center gap-1">{props.children}</span>
</span>
);
}
/** The model's name with its download's progress after it. */
function ModelProgress(props: { label: string; percent: string }) {
return (
<>
{props.label}
<span class="text-xxs font-normal">{props.percent}</span>
</>
);
}
/** 0 to 1 as a whole percentage; null, while there is no number yet, as an ellipsis. */
function percent(value: number | null): string {
return value === null ? "" : `${Math.floor(value * 100)}%`;
}
function describeModelLoad(load: ObjectMaskModelLoad): string {
const label = sam2Model(load.id).label;
switch (load.phase) {
case "download":
return load.progress === null ? `Downloading ${label}...` : `Downloading ${label}, ${Math.floor(load.progress * 100)}%`;
case "compile":
return `Preparing ${label}...`;
case "error":
return `${load.error ?? `${label} could not be loaded`}. Pick it again to retry.`;
default:
return "";
}
}
function formatSize(bytes: number): string {
return `${Math.round(bytes / 1e6)} MB`;
}
function sameState(a: SessionState | null, b: SessionState | null): boolean {
if (a === null || b === null) return a === b;
return a.status === b.status && a.completed === b.completed && a.total === b.total;
}