The VS Code of video editing, built for agents.
TypeScript
Implement Object Mask Tool Enhancements and UI Integration
konstantin-paulusb7866279fa4d
diff --git a/apps/web/src/components/sidebar-right/inspector/effects-inspector.tsx b/apps/web/src/components/sidebar-right/inspector/effects-inspector.tsxindex 2b148f23..68a31de0 100644--- a/apps/web/src/components/sidebar-right/inspector/effects-inspector.tsx+++ b/apps/web/src/components/sidebar-right/inspector/effects-inspector.tsx@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Show, createMemo } from "solid-js";+import { For, Show, createMemo, createSignal } from "solid-js";import { Button } from "@/components/ui/button";import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";import { ControlRow } from "@/components/ui/control-group";@@ -12,7 +12,15 @@ import {FloatingInspectorHeader,FloatingInspectorSeparator,} from "@/components/ui/floating-inspector";+import {+ DropdownMenu,+ DropdownMenuContent,+ DropdownMenuItem,+ DropdownMenuSeparator,+ DropdownMenuTrigger,+} from "@/components/ui/dropdown-menu";import { Icon } from "@/components/ui/icon";+import { ItemRow } from "@/components/ui/item-row";import { IncrementDecrementControl } from "@/components/ui/increment-decrement-control";import { Keyframe } from "@/components/ui/keyframe";import {@@ -26,16 +34,21 @@ import {import { SliderInput } from "@/components/ui/slider-input";import { ControlledTextField } from "@/components/ui/text-field";import { useHas, useTrait, useWorld } from "@diffusionstudio/koota-solid";-import { Computed, Effect, Hidden } from "@diffusionstudio/runtime";+import { Cache, Computed, Effect, Hidden, getParentNode } from "@diffusionstudio/runtime";import { useDerived, useEditor } from "@/engine/hooks";import { removeKeyframeTrack, syncKeyframe } from "@/engine/keyframes";+import { beginObjectMaskFor, copyObjectMask, effectHasObjectMask, getVideoRect, listObjectMasks } from "@/engine/object-mask";import { EFFECT_OPTIONS, effectOption } from "./effect-types";+import { ObjectMaskInspector } from "./object-mask";import type { EffectOption } from "./effect-types";import type { Entity } from "koota";const clampUnit = (value: number) => Math.min(1, Math.max(0, value));+// Stable identity, so an effect without masks does not resample every tick.+const NO_MASKS: Entity[] = [];+type EffectsInspectorProps = {effect: Entity;anchorRef: HTMLElement;@@ -54,12 +67,34 @@ type EffectsInspectorProps = {export function EffectsInspector(props: EffectsInspectorProps) {const world = useWorld();const editor = useEditor();+ let contentRef!: HTMLDivElement;++ const [pickedMask, setPickedMask] = createSignal<Entity>();const effect = useTrait(() => props.effect, Effect);const hidden = useHas(() => props.effect, Hidden);const option = createMemo(() => effectOption(effect()?.type));const value = useDerived(() => props.effect.get(Computed)?.value ?? 0);+ const masks = useDerived(() => props.effect.get(Cache)?.mattes ?? NO_MASKS);++ const isVideo = useDerived(() => {+ const node = getParentNode(props.effect);+ return node !== null && getVideoRect(world, node) !== null;+ });++ const shareable = useDerived(() => {+ const node = getParentNode(props.effect);+ if (!node) return [];+ return listObjectMasks(world, node).filter((source) => !effectHasObjectMask(props.effect, source));+ }, (a, b) => a.length === b.length && a.every((source, i) => source.asset.id === b[i]!.asset.id));++ const editingMask = createMemo(() => {+ const mask = pickedMask();+ return mask !== undefined && masks().includes(mask) ? mask : undefined;+ });++const editValue = (next: number) => {editor.editProperty(props.effect, "value", next);@@ -142,7 +177,7 @@ export function EffectsInspector(props: EffectsInspectorProps) {</div></FloatingInspectorHeader><FloatingInspectorSeparator />- <FloatingInspectorContent class="flex flex-col gap-2 p-4">+ <FloatingInspectorContent class="flex flex-col gap-2 p-4" ref={contentRef}><Show when={option().unit === "amount"}><ControlRow label="Amount"><SliderInput@@ -191,7 +226,103 @@ export function EffectsInspector(props: EffectsInspectorProps) {/></ControlRow></Show>++ <Show when={isVideo()}>+ <div class="flex items-center justify-between pt-2">+ <span class="text-xs text-muted-foreground">Masks</span>+ <DropdownMenu placement="bottom-end">+ <Tooltip>+ <TooltipTrigger<typeof DropdownMenuTrigger>+ as={(triggerProps: object) => (+ <DropdownMenuTrigger<typeof Button>+ {...triggerProps}+ as={(buttonProps) => (+ <Button size="icon" variant="ghost" class="text-muted-foreground" {...buttonProps}>+ <Icon name="plus-add" />+ </Button>+ )}+ />+ )}+ />+ <TooltipContent>Add mask</TooltipContent>+ </Tooltip>+ <DropdownMenuContent>+ <For each={shareable()}>+ {(source) => (+ <DropdownMenuItem onSelect={() => copyObjectMask(world, props.effect, source)}>+ {source.name}+ </DropdownMenuItem>+ )}+ </For>+ <Show when={shareable().length > 0}>+ <DropdownMenuSeparator />+ </Show>+ <DropdownMenuItem onSelect={() => beginObjectMaskFor(world, props.effect)}>+ Track new object…+ </DropdownMenuItem>+ </DropdownMenuContent>+ </DropdownMenu>+ </div>+ <For each={masks()}>+ {(mask) => (+ <ObjectMaskRow+ mask={mask}+ active={editingMask() === mask}+ onSelect={() => setPickedMask(editingMask() === mask ? undefined : mask)}+ onRemove={() => editor.remove(mask)}+ />+ )}+ </For>+ </Show></FloatingInspectorContent>++ <Show when={editingMask() !== undefined}>+ <ObjectMaskInspector+ mask={editingMask()!}+ anchorRef={contentRef}+ onClose={() => setPickedMask(undefined)}+ />+ </Show></FloatingInspector>);}++type ObjectMaskRowProps = {+ mask: Entity;+ active: boolean;+ onSelect(): void;+ onRemove(): void;+};++/** One `<mask>` of the effect; a click opens its settings. */+function ObjectMaskRow(props: ObjectMaskRowProps) {+ const hidden = useHas(() => props.mask, Hidden);++ return (+ <ItemRow+ label="Mask"+ value={"Object"}+ disabled={hidden()}+ icon={<Icon name="mask-small" />}+ class="text-foreground"+ classList={{ "bg-accent/40": props.active }}+ onClick={props.onSelect}+ >+ <Tooltip>+ <TooltipTrigger+ as={Button}+ size="icon"+ variant="ghost"+ class="text-muted-foreground"+ onClick={(event: MouseEvent) => {+ event.stopPropagation();+ props.onRemove();+ }}+ >+ <Icon name="close-remove-small" />+ </TooltipTrigger>+ <TooltipContent>Remove mask</TooltipContent>+ </Tooltip>+ </ItemRow>+ );+}diff --git a/apps/web/src/components/sidebar-right/inspector/object-mask.tsx b/apps/web/src/components/sidebar-right/inspector/object-mask.tsxnew file mode 100644index 00000000..62845f10--- /dev/null+++ b/apps/web/src/components/sidebar-right/inspector/object-mask.tsx@@ -0,0 +1,500 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Show, createResource, onMount } from "solid-js";+import { SAM2_MODELS, downloadSize, sam2Model } from "@diffusionstudio/sam2/models";+import { DEFAULT_MASK_SMOOTHING } from "@diffusionstudio/assets";+import { Button } from "@/components/ui/button";+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";+import { ControlRow } from "@/components/ui/control-group";+import {+ FloatingInspector,+ FloatingInspectorContent,+ FloatingInspectorHeader,+ FloatingInspectorSeparator,+ FloatingInspectorTitle,+} from "@/components/ui/floating-inspector";+import { Icon } from "@/components/ui/icon";+import { Keyframe } from "@/components/ui/keyframe";+import { PanelSection } from "@/components/ui/panel-section";+import { Progress } from "@/components/ui/progress";+import { SegmentedIconTabs } from "@/components/ui/segmented-icon-tabs";+import { Select, SelectContent, SelectItem, SelectPortal, SelectTrigger, SelectValue } from "@/components/ui/select";+import { SliderInput } from "@/components/ui/slider-input";+import { Switch, SwitchControl, SwitchInput, SwitchThumb } from "@/components/ui/switch";+import { useHas, useTrait, useWorld } from "@diffusionstudio/koota-solid";+import {+ AssetId, Computed, Effect, Hidden, Library, Matte, Name, VideoDecoderHandle, getParentNode,+} from "@diffusionstudio/runtime";+import { useDerived, useEditor } from "@/engine/hooks";+import { syncKeyframe } from "@/engine/keyframes";+import {+ brushRadius,+ canRestoreObjectMask,+ clearObjectTrack,+ getMaskRestore,+ getMaskRestoreOf,+ getObjectTrack,+ objectMaskMode,+ objectMaskModel,+ objectMaskModelLoad,+ pickObjectMaskModel,+ preloadObjectMaskModel,+ restoreObjectMask,+ setBrushRadius,+ setObjectMaskMode,+ targetEffect,+ trackObjectMask,+ undoMaskStroke,+} from "@/engine/object-mask";+import { effectOption } from "./effect-types";++import type { Entity } from "koota";+import type { MaskAsset } from "@diffusionstudio/assets";+import type { Sam2Model, Sam2ModelId } from "@diffusionstudio/sam2/models";+import type { MaskRestore, ObjectMaskMode, ObjectMaskModelLoad, ObjectTrackStatus } from "@/engine/object-mask";++/** The most a feather goes to from the slider, px. */+const MAX_FEATHER = 50;++const MODE_TABS = [+ { value: "points", label: "Points" },+ { value: "brush", label: "Brush" },+] as const satisfies readonly { value: ObjectMaskMode; label: string }[];++/** What each model is picked for: tiny at half resolution, the others at the full 1024. */+const MODEL_NOTES: Record<Sam2ModelId, string> = {+ tiny: "Fastest",+ small: "Finer edges",+ "base-plus": "Small objects",+ large: "Best, slowest",+};++/** The brush's size on its slider: its diameter, in percent of the frame's height. */+const BRUSH_SIZE = { min: 1, max: 30 };++type SessionState = {+ status: ObjectTrackStatus | null;+ completed: number;+ total: number;+ error: string | null;+ points: number;+ strokes: number;+};++/**+ * The object mask tool's panel, the sidebar while the tool is up: how to+ * prompt (hover a video for the segment under the pointer, click to pick),+ * where the session stands, and the button that tracks the picked object+ * through its clip and writes the mask into the document — an opacity+ * effect with a `<mask>` on the clip, whose settings then open from the+ * effect's inspector.+ */+export function ObjectMaskPanel() {+ const world = useWorld();++ const state = useDerived<SessionState | null>(() => {+ const track = getObjectTrack();+ if (!track) return null;+ return {+ status: track.status,+ completed: track.completed,+ total: track.masks.length,+ error: track.error,+ points: track.points.length,+ strokes: track.strokes.length,+ };+ }, sameState);++ const busy = () => {+ const status = state()?.status;+ return status === "loading" || status === "segmenting" || status === "tracking" || status === "saving";+ };++ // Where the mask goes: under the effect the tool was started for, else+ // under an opacity effect on the clip — the cut-out.+ const destination = () => {+ const effect = targetEffect();+ if (!effect?.isAlive()) return "The tracked mask goes under an Opacity effect on the clip: the cut-out.";+ const clip = getParentNode(effect);+ const label = effectOption(effect.get(Effect)?.type).label;+ const name = clip?.get(Name)?.value;+ return `The tracked mask goes under ${label}${name ? ` on ${name}` : ""}.`;+ };++ // The model is fetched as the tool opens, so it is ready by the first click.+ onMount(preloadObjectMaskModel);++ // The chosen model's load, while it is under way or failed; a restore's other model is not shown.+ const modelLoad = () => {+ const load = objectMaskModelLoad();+ return load && load.id === objectMaskModel() && load.phase !== "ready" ? load : null;+ };+ // A restore holds the model too: switching would take it away mid-way.+ const restoring = useDerived(() => getMaskRestore() !== null);++ // Which models are on this machine already, looked at again as loads finish.+ const [cached] = createResource(+ () => objectMaskModelLoad()?.phase ?? "idle",+ async () => (await import("@diffusionstudio/sam2")).cachedSam2Models(),+ );+ const modelTag = (model: Sam2Model) =>+ cached()?.has(model.id) ? MODEL_NOTES[model.id] : `${MODEL_NOTES[model.id]} · ${formatSize(downloadSize(model))}`;++ return (+ <PanelSection title="Object Mask">+ <ControlRow label="Model">+ <Select<Sam2Model>+ value={sam2Model(objectMaskModel())}+ options={[...SAM2_MODELS]}+ optionValue="id"+ optionTextValue="label"+ disabled={busy() || restoring()}+ onChange={(model) => model && pickObjectMaskModel(model.id)}+ itemComponent={(itemProps) => (+ <SelectItem item={itemProps.item}>+ <div class="flex min-w-0 flex-1 items-center">+ <span>{itemProps.item.rawValue.label}</span>+ <span class="ml-auto pl-3 text-xxs text-muted-foreground group-[[data-highlighted]]:text-[inherit]">+ {modelTag(itemProps.item.rawValue)}+ </span>+ </div>+ </SelectItem>+ )}+ >+ <SelectTrigger>+ <SelectValue<Sam2Model>>+ {(select) => (+ <>+ <span>{select.selectedOption()?.label}</span>+ <span class="ml-auto text-xxs text-muted-foreground">+ {select.selectedOption() && modelTag(select.selectedOption())}+ </span>+ </>+ )}+ </SelectValue>+ </SelectTrigger>+ <SelectPortal>+ <SelectContent />+ </SelectPortal>+ </Select>+ </ControlRow>+ <Show when={modelLoad()}>+ {(load) => (+ <div class="flex flex-col gap-2">+ <span class="text-xs" classList={{ "text-destructive": load().phase === "error" }}>+ {describeModelLoad(load())}+ </span>+ <Show when={load().phase === "download"}>+ <Progress value={load().progress ?? 0} minValue={0} maxValue={1} />+ </Show>+ <Show when={load().phase === "error"}>+ <div>+ <Button size="small" variant="ghost" onClick={preloadObjectMaskModel}>+ Retry+ </Button>+ </div>+ </Show>+ </div>+ )}+ </Show>+ <SegmentedIconTabs value={objectMaskMode} onChange={setObjectMaskMode} items={MODE_TABS} />+ <Show+ when={objectMaskMode() === "brush"}+ fallback={+ <p class="text-xs text-muted-foreground">+ Hover a video clip to see what a click picks. Click to mask the object on this frame; more clicks refine+ it, alt-click marks background. Then track it through the clip.+ </p>+ }+ >+ <p class="text-xs text-muted-foreground">+ Paint over the mask to add what it missed; alt-drag erases what it should not have. Click the object with+ Points first: the brush corrects the mask, and tracking follows the corrected shape.+ </p>+ <ControlRow label="Size">+ <SliderInput+ value={Math.round(brushRadius() * 200)}+ min={BRUSH_SIZE.min}+ max={BRUSH_SIZE.max}+ onChange={(value) => setBrushRadius(Math.min(BRUSH_SIZE.max, Math.max(BRUSH_SIZE.min, value)) / 200)}+ format={(value) => `${value}%`}+ />+ </ControlRow>+ </Show>+ <p class="text-xs text-muted-foreground">{destination()}</p>++ <Show when={state()}>+ {(session) => (+ <div class="flex flex-col gap-2">+ <span class="text-xs" classList={{ "text-destructive": session().status === "error" }}>+ {describe(session())}+ </span>+ <Show when={session().status === "tracking"}>+ <Progress value={progress(session())} minValue={0} maxValue={1} />+ </Show>+ </div>+ )}+ </Show>++ <div class="flex items-center gap-2">+ <Button size="small" disabled={state()?.status !== "seeded"} onClick={() => trackObjectMask(world)}>+ Track mask+ </Button>+ <Show when={state()?.status === "seeded" && state()!.strokes > 0}>+ <Button size="small" variant="ghost" onClick={() => undoMaskStroke(world)}>+ Undo stroke+ </Button>+ </Show>+ <Show when={state()}>+ <Button size="small" variant="ghost" onClick={clearObjectTrack}>+ {busy() ? "Cancel" : "Clear"}+ </Button>+ </Show>+ </div>+ </PanelSection>+ );+}++type ObjectMaskInspectorProps = {+ /** The `<mask>`; its `src` is the tracked picture. */+ mask: Entity;+ anchorRef: HTMLElement;+ onClose(): void;+};++/**+ * One `<mask>`'s settings, opened from its row in the effect's inspector the+ * way a paint opens its picker: feather, how strongly it limits the effect,+ * and inversion, plus a way to hide or remove it. A mask whose file cannot be+ * read — its decoder failed, as a video's does — can be restored from its+ * recipe.+ */+export function ObjectMaskInspector(props: ObjectMaskInspectorProps) {+ const world = useWorld();+ const editor = useEditor();++ const hidden = useHas(() => props.mask, Hidden);+ const matte = useTrait(() => props.mask, Matte);++ const maskAsset = (): MaskAsset | null => {+ const asset = world.get(Library)?.get(props.mask.get(AssetId)?.value ?? "");+ return asset?.type === "MASK" ? asset : null;+ };+ const failed = useDerived(() => props.mask.get(VideoDecoderHandle)?.errored ?? false);+ const restorable = useDerived(() => {+ const library = world.get(Library);+ const asset = maskAsset();+ return !!library && !!asset && canRestoreObjectMask(library, asset);+ });+ const restoring = useDerived(() => {+ const restore = getMaskRestoreOf(maskAsset());+ return restore ? describeRestore(restore) : null;+ });++ const restore = () => {+ const asset = maskAsset();+ if (asset) restoreObjectMask(world, asset);+ };+ const feather = useDerived(() => props.mask.get(Computed)?.blur ?? 0);+ const opacity = useDerived(() => props.mask.get(Computed)?.opacity ?? 1);++ /** Feather is the mask's blur, which is what the renderer reads it as. */+ const editFeather = (next: number) => {+ const value = Math.max(0, Math.round(next));+ editor.editProperty(props.mask, "blur", value === 0 ? false : value);+ syncKeyframe(world, editor, props.mask, "blur", value);+ };++ /** How strongly the mask limits the effect (its opacity); full strength is the prop's absence. */+ const editOpacity = (percent: number) => {+ const value = Math.min(1, Math.max(0, percent / 100));+ editor.editProperty(props.mask, "opacity", value === 1 ? false : value);+ syncKeyframe(world, editor, props.mask, "opacity", value);+ };++ const editInverted = (inverted: boolean) => {+ editor.editProperty(props.mask, "inverted", inverted ? true : false);+ };++ /** How much the edge is smoothed; the default is the prop's absence. */+ const editSmoothing = (percent: number) => {+ const value = Math.min(1, Math.max(0, Math.round(percent) / 100));+ editor.editProperty(props.mask, "smoothing", value === DEFAULT_MASK_SMOOTHING ? false : value);+ };++ const toggleHidden = () => {+ editor.editProperty(props.mask, "hidden", !hidden());+ };++ const remove = () => {+ props.onClose();+ editor.remove(props.mask);+ };++ return (+ <FloatingInspector open anchorRef={props.anchorRef} width={248}>+ <FloatingInspectorHeader class="items-center justify-between px-4">+ <FloatingInspectorTitle>Object Mask</FloatingInspectorTitle>+ <div class="flex items-center gap-1">+ <Tooltip>+ <TooltipTrigger+ as={Button}+ size="icon"+ variant="ghost"+ class="text-muted-foreground"+ onClick={toggleHidden}+ >+ <Show when={!hidden()} fallback={<Icon name="eye-off" />}>+ <Icon name="eye-on" />+ </Show>+ </TooltipTrigger>+ <TooltipContent>{hidden() ? "Show mask" : "Hide mask"}</TooltipContent>+ </Tooltip>+ <Tooltip>+ <TooltipTrigger+ as={Button}+ size="icon"+ variant="ghost"+ class="text-muted-foreground"+ onClick={props.onClose}+ >+ <Icon name="close-remove" />+ </TooltipTrigger>+ <TooltipContent>Close</TooltipContent>+ </Tooltip>+ </div>+ </FloatingInspectorHeader>+ <FloatingInspectorSeparator />+ <FloatingInspectorContent class="flex flex-col gap-2 p-4">+ <Show when={failed()}>+ <div class="flex flex-col gap-2 pb-2">+ <span class="flex items-center gap-1.5 text-xs">+ <Icon name="alert-warning" class="size-4 shrink-0 text-destructive" />+ The mask file could not be loaded.+ </span>+ <Show+ when={restoring()}+ fallback={+ <Show when={restorable()}>+ <div>+ <Button size="small" onClick={restore}>+ Restore+ </Button>+ </div>+ </Show>+ }+ >+ {(text) => <span class="text-xs text-muted-foreground">{text()}</span>}+ </Show>+ </div>+ <FloatingInspectorSeparator class="-mx-4 mb-2" />+ </Show>+ <ControlRow label="Feather">+ <SliderInput+ value={Math.round(feather())}+ min={0}+ max={MAX_FEATHER}+ onChange={editFeather}+ format={(value) => `${value}px`}+ keyframe={<Keyframe target={props.mask} property="blur" />}+ />+ </ControlRow>+ <ControlRow label="Strength">+ <SliderInput+ value={Math.round(opacity() * 100)}+ min={0}+ max={100}+ onChange={editOpacity}+ format={(value) => `${value}%`}+ keyframe={<Keyframe target={props.mask} property="opacity" />}+ />+ </ControlRow>+ <ControlRow label="Smoothing">+ <SliderInput+ value={Math.round((matte()?.smoothing ?? DEFAULT_MASK_SMOOTHING) * 100)}+ min={0}+ max={100}+ onChange={editSmoothing}+ format={(value) => `${value}%`}+ />+ </ControlRow>+ <ControlRow label="Invert">+ <Switch checked={matte()?.inverted ?? false} onChange={editInverted}>+ <SwitchInput />+ <SwitchControl variant="compact">+ <SwitchThumb variant="compact" />+ </SwitchControl>+ </Switch>+ </ControlRow>+ <div class="flex justify-end pt-1">+ <Button size="small" variant="ghost" onClick={remove}>+ Remove mask+ </Button>+ </div>+ </FloatingInspectorContent>+ </FloatingInspector>+ );+}++function describe(state: SessionState): string {+ const strokes = state.strokes === 0 ? "" : ` and ${state.strokes} ${state.strokes === 1 ? "stroke" : "strokes"}`;+ const points = `${state.points} ${state.points === 1 ? "point" : "points"}${strokes}`;+ switch (state.status) {+ case "loading":+ return "Waiting for the model...";+ case "segmenting":+ return `Masking from ${points}...`;+ case "seeded":+ return `Masked this frame from ${points}. Track it to mask the whole clip.`;+ case "tracking":+ return `Tracking, ${state.completed} of ${state.total} frames`;+ case "saving":+ return "Saving the mask...";+ case "error":+ return state.error ?? "Tracking failed";+ default:+ return "";+ }+}++function progress(state: SessionState): number {+ return state.total === 0 ? 0 : state.completed / state.total;+}++function describeModelLoad(load: ObjectMaskModelLoad): string {+ const label = sam2Model(load.id).label;+ switch (load.phase) {+ case "download":+ return load.progress === null ? `Downloading ${label}...` : `Downloading ${label}, ${Math.floor(load.progress * 100)}%`;+ case "compile":+ return `Preparing ${label}...`;+ case "error":+ return load.error ?? `${label} could not be loaded`;+ default:+ return "";+ }+}++function formatSize(bytes: number): string {+ return `${Math.round(bytes / 1e6)} MB`;+}++function describeRestore(restore: MaskRestore): string {+ switch (restore.status) {+ case "loading":+ return restore.download === null+ ? "Preparing the model..."+ : `Downloading the model, ${Math.round(restore.download * 100)}%`;+ case "tracking":+ return `Restoring, ${restore.completed} of ${restore.total} frames`;+ case "saving":+ return "Saving the mask...";+ }+}++function sameState(a: SessionState | null, b: SessionState | null): boolean {+ if (a === null || b === null) return a === b;+ return (Object.keys(a) as (keyof SessionState)[]).every((key) => a[key] === b[key]);+}diff --git a/apps/web/src/engine/hud/hud-system.ts b/apps/web/src/engine/hud/hud-system.tsindex aafefc97..06635b73 100644--- a/apps/web/src/engine/hud/hud-system.ts+++ b/apps/web/src/engine/hud/hud-system.ts@@ -20,6 +20,7 @@ import {} from '../input/interactions';import { getMarqueeQuad } from '../input/snapping';import { getMountedNameInput } from './name-input';+import { drawObjectMasks } from '../object-mask';import type { Entity, World } from 'koota';import type { Mat2D } from '@diffusionstudio/runtime';@@ -56,6 +57,7 @@ export function hudSystem(world: World): void {}drawMarquee(world, ctx, resolution);+ drawObjectMasks(world, ctx, resolution);}function drawSnapLines(world: World, ctx: Ctx2D, resolution: number): void {diff --git a/apps/web/src/engine/input/cursor.ts b/apps/web/src/engine/input/cursor.tsindex ac8cf5b4..a909260d 100644--- a/apps/web/src/engine/input/cursor.ts+++ b/apps/web/src/engine/input/cursor.ts@@ -21,7 +21,7 @@ const CORNER_SVG = `<path d='m19.7432 17.0869-4.072 4.068 2.829 2.828-8.473-.013const EDGE_SVG = `<path d='m9 17.9907v.005l5.997 5.996.001-3.999h1.999 2.02v4l5.98-6.001-5.98-5.999.001 4.019-2.021.002h-2l.001-4.022zm1.411.003 3.587-3.588-.001 2.587h3.5 2.521v-2.585l3.565 3.586-3.564 3.585-.001-2.585h-2.521l-3.499-.001-.001 2.586z' fill='%23fff'/><path d='m17.4971 18.9932h2.521v2.586l3.565-3.586-3.565-3.585v2.605h-2.521-3.5v-2.607l-3.586 3.587 3.586 3.586v-2.587z' fill='%23000'/>`;const ROTATE_SVG = `<path d="M22.4789 9.45728L25.9935 12.9942L22.4789 16.5283V14.1032C18.126 14.1502 14.6071 17.6737 14.5675 22.0283H17.05L13.513 25.543L9.97889 22.0283H12.5674C12.6071 16.5691 17.0214 12.1503 22.4789 12.1031L22.4789 9.45728Z" fill="black"/><path fill-rule="evenodd" clip-rule="evenodd" d="M21.4789 7.03223L27.4035 12.9945L21.4789 18.9521V15.1868C18.4798 15.6549 16.1113 18.0273 15.649 21.0284H19.475L13.5128 26.953L7.55519 21.0284H11.6189C12.1243 15.8155 16.2679 11.6677 21.4789 11.1559L21.4789 7.03223ZM22.4789 12.1031C17.0214 12.1503 12.6071 16.5691 12.5674 22.0284H9.97889L13.513 25.543L17.05 22.0284H14.5675C14.5705 21.6896 14.5947 21.3558 14.6386 21.0284C15.1157 17.4741 17.9266 14.6592 21.4789 14.1761C21.8063 14.1316 22.1401 14.1069 22.4789 14.1032V16.5284L25.9935 12.9942L22.4789 9.45729L22.4789 12.1031Z" fill="white"/>`;-const STATIC_CURSORS = new Set(['default', 'pointer', 'cross', 'move', 'grab', 'grabbing', 'text', 'zoom-in', 'zoom-out']);+const STATIC_CURSORS = new Set(['default', 'pointer', 'crosshair', 'move', 'grab', 'grabbing', 'text', 'zoom-in', 'zoom-out']);function getCursorCss(svg: string, rotation: number, turn: number, color: string, hx = 16, hy = 16): string {// Keep the drop shadow pointing the same way whatever the cursor's own@@ -45,7 +45,7 @@ const CURSORS: Record<string, (rotation: number, color: string) => string> = {};export type CursorType =- | 'default' | 'pointer' | 'cross' | 'grab' | 'grabbing' | 'move' | 'text'+ | 'default' | 'pointer' | 'crosshair' | 'grab' | 'grabbing' | 'move' | 'text'| 'zoom-in' | 'zoom-out'| 'ew-resize' | 'ns-resize' | 'nesw-resize' | 'nwse-resize'| 'nesw-rotate' | 'nwse-rotate' | 'swne-rotate' | 'senw-rotate';@@ -84,12 +84,12 @@ export function updateCursor(world: World, type: CursorType, rotation = 0, colorconst TOOL_CURSORS: Record<ToolType, { idle: CursorType; pressed?: CursorType }> = {[ToolType.MOVE]: { idle: 'default' },[ToolType.HAND]: { idle: 'grab', pressed: 'grabbing' },- [ToolType.BLADE]: { idle: 'cross' },- [ToolType.SCENE]: { idle: 'cross' },- [ToolType.RECT]: { idle: 'cross' },- [ToolType.TEXT]: { idle: 'cross' },+ [ToolType.BLADE]: { idle: 'crosshair' },+ [ToolType.SCENE]: { idle: 'crosshair' },+ [ToolType.RECT]: { idle: 'crosshair' },+ [ToolType.TEXT]: { idle: 'crosshair' },[ToolType.TEXT_EDIT]: { idle: 'text' },- [ToolType.OBJECT_MASK]: { idle: 'cross' },+ [ToolType.OBJECT_MASK]: { idle: 'crosshair' },};/** The cursor the armed tool asks for, pressed or at rest. */diff --git a/apps/web/src/engine/object-mask/brush.ts b/apps/web/src/engine/object-mask/brush.tsnew file mode 100644index 00000000..b735612c--- /dev/null+++ b/apps/web/src/engine/object-mask/brush.ts@@ -0,0 +1,108 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { paintMask, paintStroke } from '@diffusionstudio/sam2/mask';++import { currentSourceFrame, getVideoRect, pointOnVideo, videoPointAt } from './media';+import { brushRadius, getObjectTrack } from './store';++import type { Entity, World } from 'koota';+import type { MaskStroke, ObjectTrack } from './store';++/**+ * How far the pointer moves, in brush radii, before a stroke takes another+ * point: the segments between points are painted whole, so this only keeps+ * the recipe short.+ */+const POINT_SPACING = 0.25;++/** Stroke coordinates are kept to this many decimals, well under a pixel of the mask's grid. */+const PRECISION = 1e4;++/** The stroke under the button, and the session it paints on. */+let painting: { track: ObjectTrack; stroke: MaskStroke } | null = null;++/**+ * The session the brush can paint on over `clip`: one seeded there, showing+ * the frame it was prompted on. The brush corrects the model's mask, so the+ * object is clicked first; null until it has been.+ */+export function paintableObjectTrack(world: World, clip: Entity): ObjectTrack | null {+ const track = getObjectTrack();+ if (!track || track.clip !== clip || track.status !== 'seeded' || !track.seedMask) return null;+ return currentSourceFrame(world, clip) === track.seedFrame ? track : null;+}++/** Starts a stroke at a device-pixel point on `clip`'s video; label 1 adds to the mask, 0 erases. */+export function beginMaskStroke(world: World, clip: Entity, deviceX: number, deviceY: number, label: 0 | 1): void {+ painting = null;+ const track = paintableObjectTrack(world, clip);+ const rect = getVideoRect(world, clip);+ const point = rect && pointOnVideo(rect, deviceX, deviceY);+ if (!track || !rect || !point) return;++ const stroke: MaskStroke = { label, radius: round(brushRadius()), points: [roundPoint(point)] };+ track.strokes = [...track.strokes, stroke];+ painting = { track, stroke };+ paint(aspectOf(world, clip), 0);+}++/** Carries the stroke to a device-pixel point, wherever the drag has gone. */+export function extendMaskStroke(world: World, deviceX: number, deviceY: number): void {+ if (!painting || getObjectTrack() !== painting.track || painting.track.status !== 'seeded') {+ painting = null;+ return;+ }++ const { track, stroke } = painting;+ const rect = getVideoRect(world, track.clip);+ if (!rect) return;++ const aspect = aspectOf(world, track.clip);+ const point = roundPoint(videoPointAt(rect, deviceX, deviceY));+ const last = stroke.points[stroke.points.length - 1]!;+ if (Math.hypot((point.x - last.x) * aspect, point.y - last.y) < stroke.radius * POINT_SPACING) return;++ stroke.points.push(point);+ paint(aspect, stroke.points.length - 1);+}++export function endMaskStroke(): void {+ painting = null;+}++/** Takes the session's last stroke back. */+export function undoMaskStroke(world: World): void {+ const track = getObjectTrack();+ if (!track || track.status !== 'seeded' || !track.decoded || track.strokes.length === 0) return;++ painting = null;+ track.strokes = track.strokes.slice(0, -1);+ track.seedMask = paintMask(track.decoded, track.strokes, aspectOf(world, track.clip));+}++/**+ * Paints the growing stroke from its point `from` on into a copy of the+ * shown mask: a new mask each time, which is what the overlay redraws on.+ */+function paint(aspect: number, from: number): void {+ const { track, stroke } = painting!;+ const shown = track.seedMask!;+ const painted = { ...shown, logits: shown.logits.slice() };+ paintStroke(painted, stroke, aspect, from);+ track.seedMask = painted;+}++function aspectOf(world: World, clip: Entity): number {+ const asset = getVideoRect(world, clip)?.asset;+ return asset ? asset.width / asset.height : 1;+}++function round(value: number): number {+ return Math.round(value * PRECISION) / PRECISION;+}++function roundPoint(point: { x: number; y: number }): { x: number; y: number } {+ return { x: round(point.x), y: round(point.y) };+}diff --git a/apps/web/src/engine/object-mask/commit.tsx b/apps/web/src/engine/object-mask/commit.tsxnew file mode 100644index 00000000..e7062a5a--- /dev/null+++ b/apps/web/src/engine/object-mask/commit.tsx@@ -0,0 +1,93 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { MASK_EXTENSION, assetName, encodeMaskFile } from '@diffusionstudio/assets';+import { Effect as EffectElement, Mask } from '@diffusionstudio/reconciler';+import { Cache, Effect, EffectType, FrameRate, Hidden, Library, Name, getNextName, getParentNode } from '@diffusionstudio/runtime';+import { geometryOf } from '@diffusionstudio/sam2/models';++import { getDocumentEditor } from '../editor';+import { slug } from './copy';+import { getVideoRect } from './media';+import { getTargetEffect } from './store';++import type { Entity, World } from 'koota';+import type { MaskFrame, MaskRecipe } from '@diffusionstudio/assets';+import type { ObjectTrack } from './store';++/** Where masks go in the library: a folder per video under this one. */+const MASKS_FOLDER = 'masks';++/**+ * Turns a tracked session into the document: its frames go into the+ * library as one mask file under `masks/`, with the recipe that made them,+ * and the clip gets a `<mask>` naming it — `src` the file, `sourceIn` the+ * clip's source time its first frame belongs to — under the effect the tool+ * was started for, or else under an `opacity` effect: the cut-out. A clip+ * that already has an opacity effect gets the mask under that one. Returns+ * the mask, or null when nothing was written (the session was cancelled or+ * the clip is not a video).+ */+export async function commitObjectMask(world: World, track: ObjectTrack): Promise<Entity | null> {+ const library = world.get(Library);+ const rect = getVideoRect(world, track.clip);+ const model = track.model;+ if (!library || !rect || !model || track.masks.length === 0) return null;++ const { signal } = track.controller;+ const fps = world.get(FrameRate)?.value ?? 30;+ const name = getNextName(world, `${track.clip.get(Name)?.value || 'Video'} Mask`);+ const videoName = assetName(rect.asset).replace(/\.[^.]+$/, '');++ const blob = encodeObjectMask(rect.asset, fps, geometryOf(model.imageSize).maskSize, track.masks, {+ model: model.repo,+ source: rect.asset.id,+ first: track.first,+ seedFrame: track.seedFrame,+ points: track.points,+ strokes: track.strokes,+ });+ if (signal.aborted) return null;++ const asset = await library.store(blob, { folder: `${MASKS_FOLDER}/${slug(videoName)}`, name: `${name}.${MASK_EXTENSION}` });+ if (signal.aborted || !track.clip.isAlive()) return null;++ const editor = getDocumentEditor(world);+ const mask = () => <Mask src={asset.path} sourceIn={track.first / fps} />;++ const target = getTargetEffect();+ const holder = target && getParentNode(target) === track.clip ? target : opacityEffect(track.clip);+ if (holder) {+ const [inserted] = editor.insertElement(holder, mask);+ return inserted ?? null;+ }+ const [effect] = editor.insertElement(track.clip, () => (+ <EffectElement type="opacity" value={1}>+ {mask()}+ </EffectElement>+ ));+ return effect?.get(Cache)?.mattes[0] ?? null;+}++/** A mask file of tracked frames, on a `grid` square, over footage of `size`, one a frame at `frameRate`. */+export function encodeObjectMask(+ size: { width: number; height: number },+ frameRate: number,+ grid: number,+ masks: readonly (MaskFrame | null)[],+ recipe: MaskRecipe,+): Blob {+ return encodeMaskFile(+ { gridWidth: grid, gridHeight: grid, width: size.width, height: size.height, frameRate, recipe },+ masks,+ );+}++/** The clip's opacity effect, if it has one that is switched on: where another mask joins. */+function opacityEffect(clip: Entity): Entity | null {+ for (const effect of clip.get(Cache)?.effects ?? []) {+ if (effect.get(Effect)?.type === EffectType.OPACITY && !effect.has(Hidden)) return effect;+ }+ return null;+}diff --git a/apps/web/src/engine/object-mask/copy.tsx b/apps/web/src/engine/object-mask/copy.tsxnew file mode 100644index 00000000..04185b46--- /dev/null+++ b/apps/web/src/engine/object-mask/copy.tsx@@ -0,0 +1,84 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { assetName } from '@diffusionstudio/assets';+import { Mask } from '@diffusionstudio/reconciler';+import { AssetId, Blur, Cache, Library, getParentNode } from '@diffusionstudio/runtime';++import { getDocumentEditor } from '../editor';+import { getVideoRect } from './media';++import type { Entity, World } from 'koota';+import type { MaskAsset } from '@diffusionstudio/assets';++export function slug(name: string): string {+ return name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'mask';+}++/** A tracked mask in the library: what an effect can be given. */+export type ObjectMaskSource = {+ asset: MaskAsset;+ /** What to call it: the mask's name in the library. */+ name: string;+ /** The clip's source time its first frame belongs to, seconds. */+ sourceIn: number;+};++/**+ * The tracked masks of `clip`'s footage, from the library: every mask file+ * whose recipe names the video the clip plays, whichever effect — or none —+ * holds it now. The recipe is the record: a mask outlives the effects that+ * used it, and the library is where it is found again.+ */+export function listObjectMasks(world: World, clip: Entity): ObjectMaskSource[] {+ const library = world.get(Library);+ const rect = getVideoRect(world, clip);+ if (!library || !rect) return [];++ const sources: ObjectMaskSource[] = [];+ for (const asset of library.list()) {+ if (asset.type !== 'MASK' || asset.recipe?.source !== rect.asset.id) continue;+ sources.push({+ asset,+ name: assetName(asset).replace(/\.[^.]+$/, ''),+ sourceIn: asset.recipe.first / asset.frameRate,+ });+ }+ return sources;+}++/** Whether `effect` already has a mask of the same frames as `source`. */+export function effectHasObjectMask(effect: Entity, source: ObjectMaskSource): boolean {+ return (effect.get(Cache)?.mattes ?? []).some((mask) => mask.get(AssetId)?.value === source.asset.id);+}++/**+ * Puts the tracked frames of `source` under `effect` as a `<mask>` of its+ * own: the frames, placed at the source time they were written for. The feather carries over from another mask of the same frames on the+ * clip when there is one; inversion and strength start fresh — a mask shared+ * with a blur is usually the cut-out's inverse. Returns the new mask, or+ * null when the effect cannot be written to.+ */+export function copyObjectMask(world: World, effect: Entity, source: ObjectMaskSource): Entity | null {+ const blur = featherOf(effect, source);+ const [mask] = getDocumentEditor(world).insertElement(effect, () => (+ <Mask+ src={source.asset.path}+ {...(source.sourceIn > 0 ? { sourceIn: source.sourceIn } : {})}+ {...(blur ? { blur } : {})}+ />+ ));+ return mask ?? null;+}++/** The feather another mask of the same frames has, anywhere on the effect's clip. */+function featherOf(effect: Entity, source: ObjectMaskSource): number | undefined {+ const node = getParentNode(effect);+ for (const sibling of node?.get(Cache)?.effects ?? [effect]) {+ for (const mask of sibling.get(Cache)?.mattes ?? []) {+ if (mask.get(AssetId)?.value === source.asset.id) return mask.get(Blur)?.value;+ }+ }+ return undefined;+}diff --git a/apps/web/src/engine/object-mask/frame.ts b/apps/web/src/engine/object-mask/frame.tsnew file mode 100644index 00000000..a1675681--- /dev/null+++ b/apps/web/src/engine/object-mask/frame.ts@@ -0,0 +1,19 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { quantizeLogit } from '@diffusionstudio/assets';++import type { MaskFrame } from '@diffusionstudio/assets';+import type { Sam2Mask } from '@diffusionstudio/sam2/mask';++/**+ * The model's mask as a mask file's frame: its logits quantized, a byte a+ * cell (see `quantizeLogit`). Tracked frames are kept so, a quarter of the+ * logits' size, from the moment they arrive.+ */+export function maskFrame(mask: Sam2Mask): MaskFrame {+ const field = new Int8Array(mask.logits.length);+ for (let i = 0; i < field.length; i++) field[i] = quantizeLogit(mask.logits[i]!);+ return { field, score: mask.score, iou: mask.iou };+}diff --git a/apps/web/src/engine/object-mask/index.ts b/apps/web/src/engine/object-mask/index.tsnew file mode 100644index 00000000..4386934c--- /dev/null+++ b/apps/web/src/engine/object-mask/index.ts@@ -0,0 +1,12 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++export * from './store';+export * from './media';+export * from './tracking';+export * from './commit';+export * from './copy';+export * from './interaction';+export * from './brush';+export * from './overlay';diff --git a/apps/web/src/engine/object-mask/interaction.ts b/apps/web/src/engine/object-mask/interaction.tsnew file mode 100644index 00000000..7b720e0e--- /dev/null+++ b/apps/web/src/engine/object-mask/interaction.ts@@ -0,0 +1,65 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { RenderSurface } from '@diffusionstudio/runtime';++import { Keys, Pointer } from '../traits';+import { beginMaskStroke, endMaskStroke, extendMaskStroke } from './brush';+import { getVideoRect, pointOnVideo } from './media';+import { objectMaskMode } from './store';+import { promptObjectMask } from './tracking';++import type { World } from 'koota';+import type { DispatchedPointerEvent } from '@diffusionstudio/runtime';++/** A pointer that traveled less than this, in CSS pixels, between press and release is a click. */+const CLICK_DISTANCE = 4;++/**+ * The handler for the region the HUD lays over the tool's clip. With points,+ * a click prompts the object under the pointer; with the brush, a drag paints+ * it into the mask. With alt or shift held either marks background instead.+ */+export function handleObjectMaskInteraction(world: World, event: DispatchedPointerEvent): void {+ if (objectMaskMode() === 'brush') {+ handleBrush(world, event);+ return;+ }+ if (event.type !== 'click' || event.target.kind !== 'hud' || !event.target.entity) return;++ const pointer = world.get(Pointer);+ const resolution = world.get(RenderSurface)?.resolution ?? 1;+ if (pointer && Math.hypot(pointer.clientX - pointer.dragStartX, pointer.clientY - pointer.dragStartY) > CLICK_DISTANCE * resolution) return;++ const clip = event.target.entity;+ const rect = getVideoRect(world, clip);+ if (!rect) return;++ const point = pointOnVideo(rect, event.clientX, event.clientY);+ if (!point) return;++ promptObjectMask(world, clip, { ...point, label: backgroundHeld(world) ? 0 : 1 });+}++/** A press starts a stroke on the video, every move until the release carries it on. */+function handleBrush(world: World, event: DispatchedPointerEvent): void {+ switch (event.type) {+ case 'dragstart':+ if (event.target.kind !== 'hud' || !event.target.entity) return;+ beginMaskStroke(world, event.target.entity, event.clientX, event.clientY, backgroundHeld(world) ? 0 : 1);+ return;+ case 'drag':+ extendMaskStroke(world, event.clientX, event.clientY);+ return;+ case 'dragend':+ endMaskStroke();+ return;+ }+}++/** Whether alt or shift is held: the gesture marks background rather than the object. */+export function backgroundHeld(world: World): boolean {+ const held = world.get(Keys)?.held;+ return !!held && (held.has('alt') || held.has('shift'));+}diff --git a/apps/web/src/engine/object-mask/media.ts b/apps/web/src/engine/object-mask/media.tsnew file mode 100644index 00000000..da4944af--- /dev/null+++ b/apps/web/src/engine/object-mask/media.ts@@ -0,0 +1,77 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import {+ AssetId, Computed, Library, ScaleMode, ScaleModeType,+ entityWorldMat, findGeometryAssetSource, getScaledImageProps, getSourceWindow,+ invert2D, multiply2D, store, transformPoint, translate2D,+} from '@diffusionstudio/runtime';++import type { Entity, World } from 'koota';+import type { VideoAsset } from '@diffusionstudio/assets';+import type { Mat2D, Point } from '@diffusionstudio/runtime';++/**+ * Where a clip's video is on screen: the clip's box in device pixels, and the+ * rectangle inside it the picture is fitted to, the same fit the renderer+ * draws it with.+ */+export type VideoRect = {+ asset: VideoAsset;+ mat: Mat2D;+ width: number;+ height: number;+ x: number;+ y: number;+ w: number;+ h: number;+};++export function getVideoRect(world: World, entity: Entity): VideoRect | null {+ const source = findGeometryAssetSource(world, entity);+ if (!source) return null;++ const asset = world.get(Library)?.get(source.get(AssetId)?.value ?? '');+ if (asset?.type !== 'VIDEO') return null;++ const computed = store(world, Computed);+ const eid = entity.id();+ const width = computed.width[eid] ?? 0;+ const height = computed.height[eid] ?? 0;+ const mode = store(world, ScaleMode).value[source.id()] ?? ScaleModeType.COVER;+ const [x, y, w, h] = getScaledImageProps(mode, asset.width, asset.height, width, height);+ const mat = multiply2D(entityWorldMat(world, entity), translate2D(computed.originX[eid] ?? 0, computed.originY[eid] ?? 0));++ return { asset, mat, width, height, x, y, w, h };+}++/** A device-pixel point as 0..1 of the video frame, or null off the picture or off the clip. */+export function pointOnVideo(rect: VideoRect, deviceX: number, deviceY: number): Point | null {+ const local = transformPoint(invert2D(rect.mat), deviceX, deviceY);+ if (local.x < 0 || local.x > rect.width || local.y < 0 || local.y > rect.height) return null;++ const x = (local.x - rect.x) / rect.w;+ const y = (local.y - rect.y) / rect.h;+ if (x < 0 || x > 1 || y < 0 || y > 1) return null;++ return { x, y };+}++/** A device-pixel point in 0..1 of the video frame wherever it is, off the picture included: where a drag has gone. */+export function videoPointAt(rect: VideoRect, deviceX: number, deviceY: number): Point {+ const local = transformPoint(invert2D(rect.mat), deviceX, deviceY);+ return { x: (local.x - rect.x) / rect.w, y: (local.y - rect.y) / rect.h };+}++/** A 0..1 point on the video frame, in device pixels. */+export function videoPointToDevice(rect: VideoRect, point: Point): Point {+ return transformPoint(rect.mat, rect.x + point.x * rect.w, rect.y + point.y * rect.h);+}++/** The source frame the clip is showing, kept within the span it plays. */+export function currentSourceFrame(world: World, entity: Entity): number {+ const window = getSourceWindow(entity);+ const frame = store(world, Computed).localTime[entity.id()] ?? 0;+ return Math.min(Math.max(frame, window.in), Math.max(window.in, window.out - 1));+}diff --git a/apps/web/src/engine/object-mask/overlay.ts b/apps/web/src/engine/object-mask/overlay.tsnew file mode 100644index 00000000..ca4bfe50--- /dev/null+++ b/apps/web/src/engine/object-mask/overlay.ts@@ -0,0 +1,246 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Culled, Hidden, HitRegions, Tool, ToolType, entityQuad, isPointerInEntity } from '@diffusionstudio/runtime';+import { traceMask } from '@diffusionstudio/assets';+import { geometryOf } from '@diffusionstudio/sam2/models';++import { Pointer } from '../traits';+import { endMaskStroke, paintableObjectTrack } from './brush';+import { maskFrame } from './frame';+import { backgroundHeld, handleObjectMaskInteraction } from './interaction';+import { currentSourceFrame, getVideoRect, pointOnVideo, videoPointAt, videoPointToDevice } from './media';+import {+ brushRadius, clearTargetEffect, getObjectHover, getObjectMask, getObjectTrack, getTargetClip, objectMaskMode,+} from './store';+import { clearObjectHover, hoverObjectMask } from './tracking';++import type { Entity, World } from 'koota';+import type { MaskFrame } from '@diffusionstudio/assets';+import type { Sam2Mask } from '@diffusionstudio/sam2/mask';+import type { MaskPoint } from './store';+import type { VideoRect } from './media';++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;++const ACCENT = '#008CFF';+const BACKGROUND_POINT = '#F43535';+const MASK_ALPHA = 0.45;+const HOVER_ALPHA = 0.28;+/** The mask's edge: an accent line over a dark halo, legible on any footage; the hovered segment's is fainter. */+const OUTLINE_WIDTH = 1.5;+const OUTLINE_HALO = 3.5;+const OUTLINE_ALPHA = 1;+const HOVER_OUTLINE_ALPHA = 0.6;+const POINT_RADIUS = 5;+/** How far the pointer moves, in 0..1 of the frame, before a new segment is asked for. */+const HOVER_STEP = 0.004;++/** Traced masks, kept for scrubbing back and forth; the oldest goes first. */+const MASK_OUTLINES = 48;+const outlines = new Map<Sam2Mask | MaskFrame, Path2D>();++/** Whether the tool was up last frame, so putting it down cleans up once. */+let active = false;+let lastHover: { frame: number; x: number; y: number } | null = null;++/**+ * Everything the tool puts on the canvas while it is up: the session over its+ * clip, the segment under the pointer or the brush, the points that prompted+ * it, and the region that claims clicks on the video. Drawn last in the HUD so+ * that region sits over the selection's.+ */+export function drawObjectMasks(world: World, ctx: Ctx2D, resolution: number): void {+ if (world.get(Tool)?.value !== ToolType.OBJECT_MASK) {+ if (active) {+ active = false;+ lastHover = null;+ clearObjectHover();+ clearTargetEffect();+ endMaskStroke();+ }+ return;+ }+ active = true;++ const target = toolTarget(world);+ const brush = objectMaskMode() === 'brush';+ updateHover(world, brush ? null : target);++ const track = getObjectTrack();+ if (track && track.clip.isAlive() && !track.clip.has(Culled) && !track.clip.has(Hidden)) {+ const rect = getVideoRect(world, track.clip);+ if (rect) {+ const frame = currentSourceFrame(world, track.clip);+ const shown = getObjectMask(track, frame);+ // A tracked frame is on the grid of the model that made it; the prompted one says its own.+ const size = shown && ('size' in shown ? shown.size : track.model && geometryOf(track.model.imageSize).maskSize);+ if (shown && size) drawMask(ctx, rect, shown, size, MASK_ALPHA, OUTLINE_ALPHA, resolution);+ if (frame === track.seedFrame) drawPoints(ctx, rect, track.points, resolution);+ }+ }++ const hover = getObjectHover();+ if (hover && hover.clip.isAlive()) {+ const rect = getVideoRect(world, hover.clip);+ if (rect && hover.frame === currentSourceFrame(world, hover.clip)) drawMask(ctx, rect, hover.mask, hover.size, HOVER_ALPHA, HOVER_OUTLINE_ALPHA, resolution);+ }++ if (brush && target) drawBrush(world, ctx, target, resolution);++ if (target) {+ world.get(HitRegions)!.list.push({+ target: { kind: 'hud', id: 'object-mask', entity: target.clip, quad: entityQuad(world, target.clip) },+ callback: handleObjectMaskInteraction,+ });+ }+}++/**+ * Asks the model what is under the pointer while it rests over the tool's+ * video, so the segment a click would pick shows before the click.+ */+function updateHover(world: World, target: { clip: Entity; rect: VideoRect } | null): void {+ const pointer = world.get(Pointer);+ const point = target && pointer?.over && pointer.phase === 'lifted'+ ? pointOnVideo(target.rect, pointer.clientX, pointer.clientY)+ : null;++ if (!target || !point) {+ if (lastHover) clearObjectHover();+ lastHover = null;+ return;+ }++ const frame = currentSourceFrame(world, target.clip);+ if (lastHover && lastHover.frame === frame && Math.hypot(point.x - lastHover.x, point.y - lastHover.y) < HOVER_STEP) return;+ lastHover = { frame, x: point.x, y: point.y };+ hoverObjectMask(world, target.clip, { ...point, label: 1 });+}++/**+ * What the tool works on: the clip of the effect the tool was started for,+ * else the video clip under the pointer, where a click prompts. Null when+ * there is no video to work on.+ */+function toolTarget(world: World): { clip: Entity; rect: VideoRect } | null {+ const clip = getTargetClip() ?? videoUnderPointer(world);+ if (!clip || !clip.isAlive() || clip.has(Culled) || clip.has(Hidden)) return null;+ const rect = getVideoRect(world, clip);+ return rect ? { clip, rect } : null;+}++/** The topmost video clip under the pointer, from the regions the last frame drew. */+function videoUnderPointer(world: World): Entity | null {+ const pointer = world.get(Pointer);+ if (!pointer?.over) return null;++ const regions = world.get(HitRegions)?.list ?? [];+ const point = { x: pointer.clientX, y: pointer.clientY };+ for (let i = regions.length - 1; i >= 0; i--) {+ const { target } = regions[i]!;+ if (target.kind !== 'entity' || !target.id.isAlive()) continue;+ if (!isPointerInEntity(world, target.id, point)) continue;+ return getVideoRect(world, target.id) ? target.id : null;+ }+ return null;+}++/** Draws `mask`, on a `size` square grid, over the video. */+function drawMask(ctx: Ctx2D, rect: VideoRect, mask: Sam2Mask | MaskFrame, size: number, alpha: number, outlineAlpha: number, resolution: number): void {+ const { mat } = rect;++ // The outline is traced in grid pixels and placed on the device, then+ // filled and stroked there, so the edge is smooth and the line keeps its+ // width at any zoom. It is the edge the matte is cut along.+ const outline = new Path2D();+ const grid = new DOMMatrix([mat.a, mat.b, mat.c, mat.d, mat.e, mat.f])+ .translate(rect.x, rect.y)+ .scale(rect.w / size, rect.h / size);+ outline.addPath(maskOutline(mask, size), grid);++ ctx.save();+ ctx.setTransform(mat.a, mat.b, mat.c, mat.d, mat.e, mat.f);+ ctx.beginPath();+ ctx.rect(0, 0, rect.width, rect.height);+ ctx.clip();+ ctx.resetTransform();+ ctx.globalAlpha = alpha;+ ctx.fillStyle = ACCENT;+ ctx.fill(outline);++ ctx.globalAlpha = outlineAlpha;+ ctx.lineCap = 'round';+ ctx.lineJoin = 'round';+ ctx.lineWidth = OUTLINE_HALO * resolution;+ ctx.strokeStyle = 'rgba(0, 0, 0, 0.45)';+ ctx.stroke(outline);+ ctx.lineWidth = OUTLINE_WIDTH * resolution;+ ctx.strokeStyle = ACCENT;+ ctx.stroke(outline);+ ctx.restore();+}++/**+ * The brush's outline under the pointer, round on the footage, while there is+ * a mask on the clip to correct: red when it erases, as a background point is.+ */+function drawBrush(world: World, ctx: Ctx2D, target: { clip: Entity; rect: VideoRect }, resolution: number): void {+ const pointer = world.get(Pointer);+ if (!pointer?.over || !paintableObjectTrack(world, target.clip)) return;++ const { rect } = target;+ const pressed = pointer.phase === 'pressed';+ const point = pressed ? videoPointAt(rect, pointer.clientX, pointer.clientY) : pointOnVideo(rect, pointer.clientX, pointer.clientY);+ if (!point) return;++ const { mat } = rect;+ const radius = brushRadius() * rect.h;+ ctx.save();+ // The path is laid down in box space and stroked in device pixels, so the line keeps its width at any zoom.+ ctx.setTransform(mat.a, mat.b, mat.c, mat.d, mat.e, mat.f);+ ctx.beginPath();+ ctx.ellipse(rect.x + point.x * rect.w, rect.y + point.y * rect.h, radius, radius, 0, 0, Math.PI * 2);+ ctx.resetTransform();+ ctx.lineWidth = 3 * resolution;+ ctx.strokeStyle = 'rgba(0, 0, 0, 0.4)';+ ctx.stroke();+ ctx.lineWidth = 1.5 * resolution;+ ctx.strokeStyle = backgroundHeld(world) ? BACKGROUND_POINT : '#FFFFFF';+ ctx.stroke();+ ctx.restore();+}++function drawPoints(ctx: Ctx2D, rect: VideoRect, points: MaskPoint[], resolution: number): void {+ ctx.resetTransform();+ ctx.lineWidth = 1.5 * resolution;+ ctx.strokeStyle = '#FFFFFF';++ for (const point of points) {+ const { x, y } = videoPointToDevice(rect, point);+ ctx.beginPath();+ ctx.arc(x, y, POINT_RADIUS * resolution, 0, Math.PI * 2);+ ctx.closePath();+ ctx.fillStyle = point.label === 1 ? ACCENT : BACKGROUND_POINT;+ ctx.fill();+ ctx.stroke();+ }+}++/**+ * The mask's edge in grid cells, traced once and kept while it is shown: a+ * tracked frame as its file will hold it, the prompted one, which the brush+ * paints on, from the model's logits.+ */+function maskOutline(mask: Sam2Mask | MaskFrame, size: number): Path2D {+ let outline = outlines.get(mask);+ if (outline) return outline;++ outline = new Path2D();+ traceMask('field' in mask ? mask.field : maskFrame(mask).field, size, size, outline);+ outlines.set(mask, outline);++ if (outlines.size > MASK_OUTLINES) outlines.delete(outlines.keys().next().value!);+ return outline;+}diff --git a/apps/web/src/engine/object-mask/store.ts b/apps/web/src/engine/object-mask/store.tsnew file mode 100644index 00000000..956e3400--- /dev/null+++ b/apps/web/src/engine/object-mask/store.ts@@ -0,0 +1,246 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Tool, ToolType, getParentNode } from '@diffusionstudio/runtime';+import { DEFAULT_SAM2_MODEL, SAM2_MODELS } from '@diffusionstudio/sam2/models';+import { createSignal } from 'solid-js';++import { store } from '@/init';+import { createStoredSignal } from '@/lib/store';++import type { Entity, World } from 'koota';+import type { MaskAsset, MaskFrame } from '@diffusionstudio/assets';+import type { Sam2Mask } from '@diffusionstudio/sam2/mask';+import type { Sam2Model, Sam2ModelId } from '@diffusionstudio/sam2/models';++/** A prompt on the video, in 0..1 of its frame; label 1 marks the object, 0 marks background. */+export type MaskPoint = { x: number; y: number; label: 0 | 1 };++/**+ * A brush stroke over the prompted frame's mask, points in 0..1 of the frame+ * and radius in 0..1 of its height; label 1 adds to the mask, 0 erases.+ */+export type MaskStroke = { label: 0 | 1; radius: number; points: { x: number; y: number }[] };++/**+ * Where the session stands: the model on its way (`loading`), the prompted+ * frame being segmented (`segmenting`) or shown (`seeded`), the object+ * followed through the clip (`tracking`), its mask file written into the+ * library and the mask authored (`saving`), or stopped short.+ */+export type ObjectTrackStatus = 'loading' | 'segmenting' | 'seeded' | 'tracking' | 'saving' | 'error';++/**+ * The object mask tool's session: one object being picked on one clip. Held+ * outside the world: it is derived from the clip's footage, never authored,+ * and what it produces goes into the library and the document as an+ * `opacity` effect with a `<mask>`. There is one at a time — prompting+ * another clip, or the same clip on another frame, starts over.+ */+export type ObjectTrack = {+ /** The clip the object is picked on. */+ clip: Entity;+ /** The source frame, at project rate, the points were placed on. */+ seedFrame: number;+ points: MaskPoint[];+ /** Painted over the model's segmentation of the prompted frame, in order: what corrects it. */+ strokes: MaskStroke[];+ /** The model the masks come from, once it is loaded: what sets their grid, and what the recipe names. */+ model: Sam2Model | null;+ /** The model's segmentation of the prompted frame, before the strokes. */+ decoded: Sam2Mask | null;+ /** The prompted frame's mask, strokes and all, for the preview before tracking. */+ seedMask: Sam2Mask | null;+ /** The first source frame tracked; `masks[i]` is for frame `first + i`, as its file will hold it. */+ first: number;+ masks: (MaskFrame | null)[];+ status: ObjectTrackStatus;+ /** Frames masked so far, of `masks.length`. */+ completed: number;+ error: string | null;+ controller: AbortController;+};++/** What the pointer is over: the segment a click there would pick, on the clip and frame it was asked for. */+export type ObjectHover = {+ clip: Entity;+ frame: number;+ point: MaskPoint;+ mask: MaskFrame;+ /** The side of the mask's grid. */+ size: number;+};++// The session, reactive so the tool's panel follows it; its fields are read+// per tick, since the model writes them without events.+const [objectTrack, setTrack] = createSignal<ObjectTrack | null>(null);+let hover: ObjectHover | null = null;++/** The tool's session, reactive: null between objects. */+export { objectTrack };++export function getObjectTrack(): ObjectTrack | null {+ return objectTrack();+}++/** Starts a session, stopping whatever the one before was still computing. */+export function setObjectTrack(track: ObjectTrack): void {+ clearObjectTrack();+ setTrack(track);+}++/** The mask to show on `frame`: the tracked one once there is one, else the seed's on its own frame. */+export function getObjectMask(track: ObjectTrack, frame: number): Sam2Mask | MaskFrame | null {+ const slot = Math.round(frame - track.first);+ const tracked = track.masks[slot];+ if (tracked) return tracked;+ return frame === track.seedFrame ? track.seedMask : null;+}++/** Drops the session and stops whatever is still computing it. */+export function clearObjectTrack(): void {+ const track = objectTrack();+ if (!track) return;+ track.controller.abort();+ setTrack(null);+}++/** Drops the session when it was on `clip`: the clip is gone. */+export function clearObjectTrackOf(clip: Entity): void {+ if (objectTrack()?.clip === clip) clearObjectTrack();+}++export function clearObjectTracks(): void {+ clearObjectTrack();+ hover = null;+}++export function getObjectHover(): ObjectHover | null {+ return hover;+}++export function setObjectHover(next: ObjectHover | null): void {+ hover = next;+}++// ── How the tool prompts ─────────────────────────────────────++/** Clicks prompt the model with points; the brush paints corrections over the mask it made. */+export type ObjectMaskMode = 'points' | 'brush';++/** The brush's radius when the tool is first picked, in 0..1 of the frame's height. */+export const DEFAULT_BRUSH_RADIUS = 0.03;++const [objectMaskMode, setObjectMaskMode] = createSignal<ObjectMaskMode>('points');+const [brushRadius, setBrushRadius] = createSignal(DEFAULT_BRUSH_RADIUS);++/** The tool's mode and brush size, reactive; kept while the tool is put down. */+export { objectMaskMode, setObjectMaskMode, brushRadius, setBrushRadius };++// ── The model ────────────────────────────────────────────────++const [objectMaskModel, storeObjectMaskModel] = createStoredSignal(+ store.define<Sam2ModelId>('object-mask.model', DEFAULT_SAM2_MODEL, (id) =>+ SAM2_MODELS.some((model) => model.id === id) ? id : DEFAULT_SAM2_MODEL,+ ),+);++/** The model the tool segments and tracks with, reactive and kept across sessions. */+export { objectMaskModel };++/**+ * Makes `id` the model the tool works with (see `pickObjectMaskModel`, which+ * also loads it). The session is dropped: its masks and the frame the model+ * holds are the other model's, and the next prompt starts over with this one.+ */+export function setObjectMaskModel(id: Sam2ModelId): void {+ if (id === objectMaskModel()) return;+ clearObjectTrack();+ hover = null;+ storeObjectMaskModel(id);+}++/**+ * Where the last model asked for stands: downloading (`progress` 0 to 1),+ * compiling, ready, or failed. One model is loaded at a time, so this is+ * the one the page has, or is getting.+ */+export type ObjectMaskModelLoad = {+ id: Sam2ModelId;+ phase: 'download' | 'compile' | 'ready' | 'error';+ progress: number | null;+ error: string | null;+};++const [objectMaskModelLoad, setObjectMaskModelLoad] = createSignal<ObjectMaskModelLoad | null>(null);++/** The model's load, reactive, for the tool's panel. */+export { objectMaskModelLoad, setObjectMaskModelLoad };++// ── The effect a mask is for ─────────────────────────────────++// Set from an effect's inspector: the tool then works on that effect's clip+// and the tracked mask lands under that effect rather than under Opacity.+const [targetEffect, setTarget] = createSignal<Entity | null>(null);++/** The effect the next tracked mask goes under, reactive; null means an opacity effect on the clip. */+export { targetEffect };++export function getTargetEffect(): Entity | null {+ const effect = targetEffect();+ return effect?.isAlive() ? effect : null;+}++/** Picks the tool to add a mask to `effect`: the sidebar shows the tool's panel, aimed at the effect's clip. */+export function beginObjectMaskFor(world: World, effect: Entity): void {+ clearObjectTrack();+ setTarget(effect);+ world.set(Tool, { value: ToolType.OBJECT_MASK });+}++export function clearTargetEffect(): void {+ setTarget(null);+}++/** The clip the tool is aimed at by a target effect, or null when it is free to pick any video. */+export function getTargetClip(): Entity | null {+ const effect = getTargetEffect();+ return effect ? getParentNode(effect) : null;+}++// ── Restoring a mask from its recipe ─────────────────────────++/**+ * A mask whose file is being made again from its recipe (see+ * `restoreObjectMask`): the model on its way, the object tracked again, the+ * file written back. One at a time — the model holds one object — and, like+ * the session, its fields are read per tick.+ */+export type MaskRestore = {+ asset: MaskAsset;+ status: 'loading' | 'tracking' | 'saving';+ /** Frames masked so far, of `total`. */+ completed: number;+ total: number;+ /** Model download, 0 to 1, while it is on its way. */+ download: number | null;+ controller: AbortController;+};++let maskRestore: MaskRestore | null = null;++export function getMaskRestore(): MaskRestore | null {+ return maskRestore;+}++/** Starts a restore, stopping the one before. */+export function setMaskRestore(restore: MaskRestore): void {+ maskRestore?.controller.abort();+ maskRestore = restore;+}++/** Drops `restore` once it is over, unless another has taken its place. */+export function finishMaskRestore(restore: MaskRestore): void {+ if (maskRestore === restore) maskRestore = null;+}diff --git a/apps/web/src/engine/object-mask/tracking.ts b/apps/web/src/engine/object-mask/tracking.tsnew file mode 100644index 00000000..b150d97f--- /dev/null+++ b/apps/web/src/engine/object-mask/tracking.ts@@ -0,0 +1,423 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { isProjectSource } from '@diffusionstudio/assets';+import { FrameRate, Library, Tool, ToolType, getSourceWindow, getVideoTrack } from '@diffusionstudio/runtime';+import { paintMask } from '@diffusionstudio/sam2/mask';+import { sam2ModelOfRepo } from '@diffusionstudio/sam2/models';+import { toast } from 'somoto';++import { getDocumentEditor } from '../editor';+import { commitObjectMask, encodeObjectMask } from './commit';+import { maskFrame } from './frame';+import { currentSourceFrame, getVideoRect } from './media';+import {+ clearObjectTrack, clearTargetEffect, finishMaskRestore, getMaskRestore, getObjectTrack, objectMaskModel, objectMaskModelLoad,+ setMaskRestore, setObjectHover, setObjectMaskModel, setObjectMaskModelLoad, setObjectTrack,+} from './store';++import type { Entity, World } from 'koota';+import type { InputVideoTrack } from 'mediabunny';+import type { Sam2Mask } from '@diffusionstudio/sam2/mask';+import type { Sam2Video } from '@diffusionstudio/sam2';+import type { Sam2ModelId } from '@diffusionstudio/sam2/models';+import type { AssetLibrary, MaskAsset, MaskFrame, VideoAsset } from '@diffusionstudio/assets';+import type { MaskPoint, MaskRestore, MaskStroke, ObjectMaskModelLoad, ObjectTrack } from './store';++// The model holds one object's memory and one held frame, so requests run+// one after another.+let queue: Promise<void> = Promise.resolve();+/** Whether a prompt or a track has the model; hovers wait for neither. */+let busy = false;++/** What every step needs: the model, the clip's decodable track, and where its frames are. */+type Session = {+ model: Sam2Video;+ video: InputVideoTrack;+ /** Where a project-rate source frame is in the file, in seconds. */+ seconds: (frame: number) => number;+ /** The footage's width over its height, which brush strokes are round in. */+ aspect: number;+};++/** The frame the model holds encoded, by clip and source frame, so prompts on it cost the decoder alone. */+let held: { clip: Entity; frame: number } | null = null;++/**+ * Prompts the object on `clip` with one more point and segments the current+ * frame. A point on the clip and frame the session was prompted on refines+ * that object, its brush strokes painted over the new segmentation; a point+ * anywhere else starts over there. Tracking through the clip waits for+ * `trackObjectMask`.+ */+export function promptObjectMask(world: World, clip: Entity, point: MaskPoint): void {+ const frame = currentSourceFrame(world, clip);+ const existing = getObjectTrack();+ const same = existing?.clip === clip && existing.seedFrame === frame;+ const points = same ? [...existing.points, point] : [point];++ const track: ObjectTrack = {+ clip,+ seedFrame: frame,+ points,+ strokes: same ? [...existing.strokes] : [],+ model: null,+ decoded: null,+ seedMask: null,+ first: frame,+ masks: [],+ status: 'loading',+ completed: 0,+ error: null,+ controller: new AbortController(),+ };+ setObjectTrack(track);+ setObjectHover(null);++ enqueue(world, track, async (session) => {+ track.status = 'segmenting';+ await holdFrame(session, clip, track.seedFrame);+ session.model.reset();+ const decoded = await session.model.seedHeld(track.points, 0);+ if (track.controller.signal.aborted) return;+ track.decoded = decoded;+ track.seedMask = paintMask(decoded, track.strokes, session.aspect);+ track.status = 'seeded';+ });+}++/**+ * Follows the prompted object through every source frame the clip plays,+ * then writes the frames into the library and authors the mask on the clip,+ * under an opacity effect (see `commitObjectMask`). The session is spent+ * once the document holds the mask: the clip is selected with the Move tool+ * so its effects, and the mask under them, are at hand.+ */+export function trackObjectMask(world: World): void {+ const seeded = getObjectTrack();+ if (!seeded || seeded.status !== 'seeded') return;++ const track: ObjectTrack = { ...seeded, controller: new AbortController(), status: 'loading', error: null };+ setObjectTrack(track);+ setObjectHover(null);++ enqueue(world, track, async (session) => {+ const { signal } = track.controller;+ const { trackObject } = await import('@diffusionstudio/sam2');++ // Every source frame of the clip, masked outward from the prompted one.+ const window = getSourceWindow(track.clip);+ const first = window.in;+ const count = Math.max(window.out - 1, first) - first + 1;+ const seedIndex = Math.min(Math.max(track.seedFrame - first, 0), count - 1);++ Object.assign(track, { first, masks: new Array<MaskFrame | null>(count).fill(null), completed: 0, status: 'tracking' });+ releaseFrame(session);+ session.model.reset();+ await trackObject(session.model, {+ track: session.video,+ timestamps: track.masks.map((_, i) => session.seconds(first + i)),+ seedIndex,+ points: track.points,+ correct: corrector(track.strokes, session.aspect),+ signal,+ onMask: (index, frameMask) => {+ track.masks[index] = maskFrame(frameMask);+ track.completed++;+ },+ });+ if (signal.aborted) return;++ track.status = 'saving';+ const mask = await commitObjectMask(world, track);+ if (signal.aborted) return;++ clearObjectTrack();+ clearTargetEffect();+ if (!mask) {+ toast.error('Object mask failed', { description: 'The mask could not be saved.' });+ return;+ }+ if (track.clip.isAlive()) getDocumentEditor(world).select(track.clip);+ if (world.get(Tool)?.value === ToolType.OBJECT_MASK) world.set(Tool, { value: ToolType.MOVE });+ });+}++// ── Hover ────────────────────────────────────────────────────++/** The latest point the pointer asked about while a preview was running; served next. */+let pendingHover: { clip: Entity; point: MaskPoint } | null = null;+let hovering = false;++/**+ * Previews the segment a click at `point` on `clip` would pick, on the+ * current frame, kept hot in the model so only the mask decoder runs. On the+ * session's own clip and frame the hover refines its prompt. Requests+ * coalesce: while one runs the newest point waits, and the session's own+ * work (segmenting, tracking) takes precedence over hovering.+ */+export function hoverObjectMask(world: World, clip: Entity, point: MaskPoint): void {+ pendingHover = { clip, point };+ if (hovering || busy) return;+ hovering = true;++ queue = queue.then(async () => {+ try {+ while (pendingHover && !busy) {+ const { clip, point: at } = pendingHover;+ pendingHover = null;+ if (!clip.isAlive()) continue;++ const track = getObjectTrack();+ if (track && track.status !== 'seeded' && track.status !== 'error') continue;++ const frame = currentSourceFrame(world, clip);+ const session = await openClipSession(world, clip);+ await holdFrame(session, clip, frame);+ // On the prompted frame the hover refines the prompt, elsewhere it starts one.+ const refining = track?.clip === clip && track.seedFrame === frame;+ const preview = refining+ ? paintMask(await session.model.preview([...track.points, at]), track.strokes, session.aspect)+ : await session.model.preview([at]);+ setObjectHover({ clip, frame, point: at, mask: maskFrame(preview), size: preview.size });+ }+ } catch {+ // A hover is a hint; a failed one is nothing.+ } finally {+ hovering = false;+ }+ });+}++/** Forgets the hovered segment, when the pointer leaves the video. */+export function clearObjectHover(): void {+ pendingHover = null;+ setObjectHover(null);+}++// ── Restore ──────────────────────────────────────────────────++/**+ * Whether `asset` is a mask that can be made again: it has its recipe, the+ * footage the recipe names is in the library, and its file lives in the+ * project, where the app may write.+ */+export function canRestoreObjectMask(library: AssetLibrary, asset: MaskAsset): boolean {+ const recipe = asset.recipe;+ return !!recipe && library.get(recipe.source)?.type === 'VIDEO' && isProjectSource(asset.source);+}++/**+ * Makes a mask's frames again from its recipe, for a mask whose file went+ * missing: the same footage, prompt and span, at the rate the file was+ * written at, tracked again — by the model that made them, or the tool's+ * when that one is no longer offered — and written back where the file was.+ * The library takes the file in again at the same path, so every `<mask>`+ * naming it shows it; should the new frames differ from the lost ones, the+ * new id is followed to them (see `onRelink`). The model's own sessions come first — a restore+ * waits its turn, and stops when another is asked for.+ */+export function restoreObjectMask(world: World, asset: MaskAsset): void {+ const library = world.get(Library);+ const recipe = asset.recipe;+ if (!library || !recipe || !canRestoreObjectMask(library, asset)) return;+ const video = library.get(recipe.source) as VideoAsset;++ const total = Math.max(1, Math.round(asset.duration * asset.frameRate));+ const restore: MaskRestore = {+ asset,+ status: 'loading',+ completed: 0,+ total,+ download: null,+ controller: new AbortController(),+ };+ setMaskRestore(restore);++ queue = queue.then(async () => {+ const { signal } = restore.controller;+ if (signal.aborted) return;++ busy = true;+ try {+ const model = sam2ModelOfRepo(recipe.model)?.id ?? objectMaskModel();+ const session = await openSession(video, asset.frameRate, model, (download) => (restore.download = download));+ if (signal.aborted) return;+ const { trackObject } = await import('@diffusionstudio/sam2');++ const masks = new Array<MaskFrame | null>(total).fill(null);+ restore.status = 'tracking';+ releaseFrame(session);+ session.model.reset();+ await trackObject(session.model, {+ track: session.video,+ timestamps: masks.map((_, i) => session.seconds(recipe.first + i)),+ seedIndex: Math.min(Math.max(recipe.seedFrame - recipe.first, 0), total - 1),+ points: recipe.points,+ correct: corrector(recipe.strokes ?? [], session.aspect),+ signal,+ onMask: (index, mask) => {+ masks[index] = maskFrame(mask);+ restore.completed++;+ },+ });+ if (signal.aborted) return;++ restore.status = 'saving';+ const file = encodeObjectMask(asset, asset.frameRate, session.model.maskSize, masks, { ...recipe, model: session.model.model.repo });+ await library.fs.write(asset.source, file);+ await library.relink(asset, asset.source);+ } catch (error) {+ if (signal.aborted) return;+ toast.error('Mask restore failed', { description: error instanceof Error ? error.message : String(error) });+ } finally {+ finishMaskRestore(restore);+ busy = false;+ // A restore by another model has replaced the tool's; it is loaded again.+ if (preloaded) preloadObjectMaskModel();+ }+ });+}++/** The restore of `asset` while it runs; null when there is none. */+export function getMaskRestoreOf(asset: MaskAsset | null | undefined): MaskRestore | null {+ const restore = getMaskRestore();+ return asset && restore?.asset.id === asset.id ? restore : null;+}++// ── The model ────────────────────────────────────────────────────++/** Whether the tool has been opened, and so wants its model loaded. */+let preloaded = false;++/**+ * Loads the tool's model ahead of the first click, as the tool opens: the+ * download starts before the video is touched. Its progress, or what went+ * wrong, is on `objectMaskModelLoad`.+ */+export function preloadObjectMaskModel(): void {+ preloaded = true;+ loadModel(objectMaskModel()).catch(() => undefined);+}++/**+ * Switches the tool to model `id` and starts loading it; a model still+ * downloading is cancelled, and the files it fetched whole stay cached.+ */+export function pickObjectMaskModel(id: Sam2ModelId): void {+ setObjectMaskModel(id);+ preloadObjectMaskModel();+}++/**+ * Model `id`, loaded once and shared by whoever asks, with its load on+ * `objectMaskModelLoad`; `onDownload` hears its download, 0 to 1.+ */+async function loadModel(id: Sam2ModelId, onDownload?: (progress: number | null) => void): Promise<Sam2Video> {+ const current = objectMaskModelLoad();+ if (current?.id !== id || current.phase === 'error') {+ setObjectMaskModelLoad({ id, phase: 'download', progress: null, error: null });+ }+ // Another model asked for since takes the panel over.+ const update = (next: Omit<ObjectMaskModelLoad, 'id'>) => {+ if (objectMaskModelLoad()?.id === id) setObjectMaskModelLoad({ id, ...next });+ };++ try {+ const { loadSam2 } = await import('@diffusionstudio/sam2');+ const model = await loadSam2(id, {+ onProgress: (progress) => {+ const fraction = progress.phase === 'download' ? progress.loaded / progress.total : null;+ onDownload?.(fraction);+ // A percent at a time: the download reports every chunk.+ const shown = objectMaskModelLoad();+ if (fraction !== null && shown?.phase === 'download' && shown.progress !== null && Math.floor(shown.progress * 100) === Math.floor(fraction * 100)) return;+ update({ phase: progress.phase, progress: fraction, error: null });+ },+ });+ update({ phase: 'ready', progress: null, error: null });+ return model;+ } catch (error) {+ // A cancelled load gave way to another model, which has the panel now.+ if (!(error instanceof DOMException && error.name === 'AbortError')) {+ update({ phase: 'error', progress: null, error: error instanceof Error ? error.message : String(error) });+ }+ throw error;+ }+}++/** Runs `step` for `track` once the model is free, and files whatever goes wrong on the track. */+function enqueue(world: World, track: ObjectTrack, step: (session: Session) => Promise<void>): void {+ queue = queue.then(async () => {+ const { signal } = track.controller;+ if (signal.aborted) return;++ busy = true;+ try {+ const session = await openClipSession(world, track.clip);+ if (signal.aborted) return;+ track.model = session.model.model;+ await step(session);+ } catch (error) {+ if (signal.aborted) return;+ track.status = 'error';+ track.error = error instanceof Error ? error.message : String(error);+ toast.error('Object mask failed', { description: track.error });+ } finally {+ busy = false;+ }+ });+}++/** A session with the tool's model on the footage `clip` plays, at the project's rate. */+async function openClipSession(world: World, clip: Entity): Promise<Session> {+ const rect = getVideoRect(world, clip);+ if (!rect) throw new Error('The selection is not a video');+ const fps = world.get(FrameRate)?.value ?? 30;+ return openSession(rect.asset, fps, objectMaskModel());+}++/**+ * The model `id`, and `asset`'s footage on a grid of `fps` source frames.+ * Another model than the one loaded replaces it, and whatever it held.+ */+async function openSession(asset: VideoAsset, fps: number, id: Sam2ModelId, onDownload?: (progress: number | null) => void): Promise<Session> {+ const model = await loadModel(id, onDownload);++ const video = await getVideoTrack(asset);+ if (!video) throw new Error('The video could not be decoded');++ const firstTimestamp = await video.getFirstTimestamp();+ return {+ model,+ video,+ seconds: (frame) => sourceSeconds(frame, fps, asset, firstTimestamp),+ aspect: asset.width / asset.height,+ };+}++/** What has the model remember the seed's mask with `strokes` painted over it; none when there are none. */+function corrector(strokes: readonly MaskStroke[], aspect: number): ((mask: Sam2Mask) => Sam2Mask) | undefined {+ return strokes.length > 0 ? (mask) => paintMask(mask, strokes, aspect) : undefined;+}++/** Has the model hold `frame` of `clip`, unless it already does. */+async function holdFrame(session: Session, clip: Entity, frame: number): Promise<void> {+ if (held && held.clip === clip && held.frame === frame && session.model.holding) return;+ const { holdFrame: hold } = await import('@diffusionstudio/sam2');+ await hold(session.model, { track: session.video, timestamp: session.seconds(frame) });+ held = { clip, frame };+}++function releaseFrame(session: Session): void {+ session.model.release();+ held = null;+}++/** Where a project-rate source frame is in the file, on the file's own frame grid, as the preview decoder seeks it. */+function sourceSeconds(frame: number, fps: number, asset: VideoAsset, firstTimestamp: number): number {+ return Math.round((frame / fps) * asset.frameRate) / asset.frameRate + firstTimestamp;+}++export { clearObjectTrack };diff --git a/docs/reference/jsx/styles.md b/docs/reference/jsx/styles.mdindex 4f69757b..53d1963b 100644--- a/docs/reference/jsx/styles.md+++ b/docs/reference/jsx/styles.md@@ -80,6 +80,7 @@ Under an `"opacity"` effect the mask is the cut-out: the clip shows inside the m| `blur` | `number` | `0` | Feather: radius in px the edge falls off over. || `opacity` | `number` | `1` | How strongly the mask limits the effect. `1` stops the effect at the edge; lower lets that much of it through outside; `0` is no mask. || `inverted` | `boolean` | `false` | Covers what the picture does not instead. |+| `smoothing` | `number` | `0.25` | How much a mask file's edge is smoothed, `0`–`1`. At `0` it is exactly where the model put it; the more, the rounder, and the more of what is thin or small melts away. Sharp at any size either way. || `hidden` | `boolean` | absent | Switches the mask off without removing it. |`blur` and `opacity` take a [`<keyframeTrack>`](./keyframes.md).diff --git a/packages/assets/src/derive/thumbnail.ts b/packages/assets/src/derive/thumbnail.tsindex c08f7a33..4caca771 100644--- a/packages/assets/src/derive/thumbnail.ts+++ b/packages/assets/src/derive/thumbnail.ts@@ -2,13 +2,17 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// The thumbnail of an image or video file: scaled to a width, its aspect-// ratio kept, encoded as WebP. For a video it is the frame shortly after the-// start. The decoder does the scaling (mediabunny's CanvasSink for video,-// createImageBitmap for images), so the picture is encoded once, as it comes.+// The thumbnail of an image, video or mask file: scaled to a width, its+// aspect ratio kept, encoded as WebP. For a video it is the frame shortly+// after the start; for a mask, its middle tracked frame. The decoder does+// the scaling (mediabunny's CanvasSink for video, createImageBitmap for+// images), so the picture is encoded once, as it comes.import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';+import { MASK_MIME_TYPE, MaskFile } from '../mask';+import { traceMask } from '../mask-outline';+/** The width the asset bar shows; what the cache stores without a variant. */export const DEFAULT_THUMBNAIL_WIDTH = 300;@@ -17,10 +21,15 @@ const VIDEO_THUMBNAIL_OFFSET = 0.3;const WEBP_QUALITY = 0.7;-/** A WebP thumbnail of `file`, `width` wide, or null when it is neither an image nor a video with a picture. */+/** A mask thumbnail's object and background: white on near-black, as a matte reads. */+const MASK_FOREGROUND = '#ffffff';+const MASK_BACKGROUND = '#1a1a1a';++/** A WebP thumbnail of `file`, `width` wide, or null when it is no image, video with a picture, or mask. */export async function deriveThumbnail(file: Blob, mimeType: string, width = DEFAULT_THUMBNAIL_WIDTH): Promise<Blob | null> {if (mimeType.startsWith('image/')) return imageThumbnail(file, width);if (mimeType.startsWith('video/')) return videoThumbnail(file, width);+ if (mimeType === MASK_MIME_TYPE) return maskThumbnail(file, width);return null;}@@ -52,6 +61,30 @@ async function videoThumbnail(file: Blob, width: number): Promise<Blob | null> {}}+async function maskThumbnail(file: Blob, width: number): Promise<Blob | null> {+ const mask = await MaskFile.read(file);+ const { gridWidth, gridHeight, width: frameWidth, height: frameHeight, frameCount } = mask.header;+ const tracked: number[] = [];+ for (let i = 0; i < frameCount; i++) if (mask.has(i)) tracked.push(i);+ const middle = tracked[Math.floor(tracked.length / 2)];+ if (middle === undefined || !gridWidth || !gridHeight) return null;++ const field = new Int8Array(gridWidth * gridHeight);+ mask.field(middle, field);+ const outline = new Path2D();+ traceMask(field, gridWidth, gridHeight, outline);++ const height = Math.max(1, Math.round((width * (frameHeight || gridHeight)) / (frameWidth || gridWidth)));+ const canvas = new OffscreenCanvas(width, height);+ const ctx = canvas.getContext('2d')!;+ ctx.fillStyle = MASK_BACKGROUND;+ ctx.fillRect(0, 0, width, height);+ ctx.scale(width / gridWidth, height / gridHeight);+ ctx.fillStyle = MASK_FOREGROUND;+ ctx.fill(outline);+ return encode(canvas);+}+function encode(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob | null> {if (canvas instanceof OffscreenCanvas) return canvas.convertToBlob({ type: 'image/webp', quality: WEBP_QUALITY });return new Promise((resolve) => canvas.toBlob(resolve, 'image/webp', WEBP_QUALITY));diff --git a/packages/assets/src/index.ts b/packages/assets/src/index.tsindex ef9f6294..a4b8349a 100644--- a/packages/assets/src/index.ts+++ b/packages/assets/src/index.ts@@ -7,6 +7,8 @@ export * from './manifest';export * from './fs';export * from './hash';export * from './probe';+export * from './mask';+export * from './mask-outline';export * from './library';export * from './browser';export * from './cache';diff --git a/packages/assets/src/mask-outline.ts b/packages/assets/src/mask-outline.tsnew file mode 100644index 00000000..cea0e962--- /dev/null+++ b/packages/assets/src/mask-outline.ts@@ -0,0 +1,338 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { FIELD_LOGITS, MASK_FIELD_MAX } from './mask';++// A mask's edge as a smooth outline: a path, which a canvas fills with+// anti-aliasing at whatever size the mask lands at, however coarse the grid.+//+// The edge is where the model put it. A frame's field is the model's logits+// on the decoder's own grid (see `MaskFile`), which the model upsamples+// twice, bilinearly, before it cuts at zero; so is it here, and the edge is+// traced where the upsampled logits cross zero — marching squares over the+// cell centers, each crossing placed between its two centers by the logits,+// to a small fraction of a cell. The crossings are chained into closed loops+// and drawn as quadratic B-splines.+//+// Smoothing is a look on top: the logits blurred before they are upsampled+// and cut, and the loops relaxed before they are drawn. At 0 the edge is the model's; the+// more, the rounder, and the more of what is thin or small melts away.+//+// The loops are oriented, the object on the same side of every one, so holes+// run the other way and the path fills right under the nonzero rule.++/** What `traceMask` draws into: a `Path2D`, or a 2D context. */+export type MaskOutlineSink = {+ moveTo(x: number, y: number): void;+ lineTo(x: number, y: number): void;+ quadraticCurveTo(cpx: number, cpy: number, x: number, y: number): void;+ closePath(): void;+};++/** The smoothing a mask gets unless it asks for another. */+export const DEFAULT_MASK_SMOOTHING = 0.25;++/** The blur's standard deviation at full smoothing, in cells of the field. */+const MAX_BLUR_SIGMA = 1;++/** How many times each loop is relaxed at full smoothing. */+const MAX_RELAX_PASSES = 8;++/** The logit of a saturated cell: far enough in, or out, that the field says no more. */+const SATURATED = FIELD_LOGITS[2 * MASK_FIELD_MAX]!;++/** The cell a field byte of `q` holds, as its logit. */+const logitOf = (q: number) => FIELD_LOGITS[q + MASK_FIELD_MAX]!;++/**+ * Traces the edge of `field` — a `width` by `height` grid of field bytes,+ * row-major (see `quantizeLogit`) — into `path` as closed loops, in grid+ * cells: cell (x, y) covers x..x+1, y..y+1. `smoothing`, 0 to 1, rounds it+ * off. Where the mask touches the grid's border the loop runs along it,+ * square at its corners.+ */+export function traceMask(+ field: ArrayLike<number>,+ width: number,+ height: number,+ path: MaskOutlineSink,+ smoothing = DEFAULT_MASK_SMOOTHING,+): void {+ if (width === 0 || height === 0) return;+ const amount = Math.min(1, Math.max(0, smoothing));++ // Everything below is on the upsampled grid, padded by a cell saturated out+ // on every side, so a loop closes around the object where it touches the border.+ const w = 2 * width;+ const h = 2 * height;+ const stride = w + 2;+ const logits = upsample(blur(dequantize(field), width, height, amount * MAX_BLUR_SIGMA), width, height);+ const inside = (k: number) => logits[k]! > 0;++ // Crossings are keyed by the lattice edge they lie on: 2k for the edge from+ // center k to the one right of it, 2k + 1 for the one below.+ const next = new Int32Array(2 * stride * (h + 2)).fill(-1);+ for (let y = 0; y <= h; y++) {+ for (let x = 0; x <= w; x++) {+ const k = y * stride + x;+ const index = (inside(k) ? 8 : 0) | (inside(k + 1) ? 4 : 0) | (inside(k + stride + 1) ? 2 : 0) | (inside(k + stride) ? 1 : 0);+ if (index === 0 || index === 15) continue;+ // A saddle joins the object across the cell when its middle is in.+ const joined = (index !== 5 && index !== 10) || logits[k]! + logits[k + 1]! + logits[k + stride]! + logits[k + stride + 1]! > 0;+ const segments = (joined ? SEGMENTS : SPLIT_SEGMENTS)[index]!;+ // The cell's sides as crossings: top, right, bottom, left.+ const sides = [2 * k, 2 * (k + 1) + 1, 2 * (k + stride), 2 * k + 1];+ for (let i = 0; i < segments.length; i += 2) next[sides[segments[i]!]!] = sides[segments[i + 1]!]!;+ }+ }++ const xs: number[] = [];+ const ys: number[] = [];+ for (let start = 0; start < next.length; start++) {+ if (next[start]! < 0) continue;+ xs.length = 0;+ ys.length = 0;+ // Each crossing is unlinked as it is taken, so the walk stops back at its start.+ let id = start;+ while (id >= 0 && next[id]! >= 0) {+ const k = id >> 1;+ const cx = (k % stride) - 1;+ const cy = Math.floor(k / stride) - 1;+ const down = (id & 1) === 1;+ // Center k is at (cx + 0.5, cy + 0.5); the crossing is where the logits+ // reach zero on the way to its neighbor, or on the border when the+ // neighbor is padding.+ const j = down ? k + stride : k + 1;+ const padding = down ? cy < 0 || cy >= h - 1 : cx < 0 || cx >= w - 1;+ const t = padding ? 0.5 : -logits[k]! / (logits[j]! - logits[k]!);+ xs.push(cx + 0.5 + (down ? 0 : t));+ ys.push(cy + 0.5 + (down ? t : 0));+ const following = next[id]!;+ next[id] = -1;+ id = following;+ }+ drawLoop(xs, ys, w, h, path, amount * MAX_RELAX_PASSES, 0.5);+ }+}++/** The logits `field` stands for. */+function dequantize(field: ArrayLike<number>): Float32Array {+ const logits = new Float32Array(field.length);+ for (let i = 0; i < field.length; i++) logits[i] = logitOf(field[i]!);+ return logits;+}++/**+ * `logits`, `width` by `height`, upsampled twice into the padded grid, as the+ * model's decoder does it: bilinear, cell centers aligned, the border held.+ * Each new cell is three quarters the nearest old one and a quarter the next.+ */+function upsample(logits: Float32Array, width: number, height: number): Float32Array {+ const w = 2 * width;+ const rows = new Float32Array(w * height);+ for (let y = 0; y < height; y++) {+ const row = y * width;+ for (let x = 0; x < width; x++) {+ const center = logits[row + x]!;+ const left = logits[row + Math.max(x - 1, 0)]!;+ const right = logits[row + Math.min(x + 1, width - 1)]!;+ rows[y * w + 2 * x] = 0.25 * left + 0.75 * center;+ rows[y * w + 2 * x + 1] = 0.75 * center + 0.25 * right;+ }+ }++ const stride = w + 2;+ const out = new Float32Array(stride * (2 * height + 2)).fill(-SATURATED);+ for (let y = 0; y < height; y++) {+ const above = Math.max(y - 1, 0) * w;+ const below = Math.min(y + 1, height - 1) * w;+ const top = (2 * y + 1) * stride + 1;+ const bottom = top + stride;+ for (let x = 0; x < w; x++) {+ const center = rows[y * w + x]!;+ out[top + x] = 0.25 * rows[above + x]! + 0.75 * center;+ out[bottom + x] = 0.75 * center + 0.25 * rows[below + x]!;+ }+ }+ return out;+}++/**+ * `logits`, `w` by `h`, blurred by a Gaussian of `sigma` cells. Past the+ * grid's border the blur takes the nearest cell, so an object cut off by the+ * frame stays full to its edge. A cell whose whole window is saturated one+ * way stays so without being weighed, so the blur costs what the edge's+ * length does, not the grid's area.+ */+function blur(logits: Float32Array, w: number, h: number, sigma: number): Float32Array {+ const { radius, weights } = gaussian(sigma);+ if (radius === 0) return logits;++ // Each line is copied into `line` with its end cells repeated `radius`+ // beyond, blurred into `blurred`, and written back.+ const longest = Math.max(w, h);+ const line = new Float32Array(longest + 2 * radius);+ const blurred = new Float32Array(longest);+ // How many cells of `line` before each are not saturated out, and not in.+ const notOut = new Int32Array(longest + 2 * radius + 1);+ const notIn = new Int32Array(longest + 2 * radius + 1);+ const taps = 2 * radius + 1;++ const blurLine = (length: number) => {+ const first = line[radius]!;+ const last = line[radius + length - 1]!;+ for (let i = 0; i < radius; i++) {+ line[i] = first;+ line[radius + length + i] = last;+ }+ const padded = length + 2 * radius;+ for (let i = 0; i < padded; i++) {+ const value = line[i]!;+ notOut[i + 1] = notOut[i]! + (value !== -SATURATED ? 1 : 0);+ notIn[i + 1] = notIn[i]! + (value !== SATURATED ? 1 : 0);+ }+ for (let i = 0; i < length; i++) {+ if (notOut[i + taps] === notOut[i]) blurred[i] = -SATURATED;+ else if (notIn[i + taps] === notIn[i]) blurred[i] = SATURATED;+ else {+ let sum = 0;+ for (let j = 0; j < taps; j++) sum += line[i + j]! * weights[j]!;+ blurred[i] = sum;+ }+ }+ };++ const rows = new Float32Array(w * h);+ for (let y = 0; y < h; y++) {+ line.set(logits.subarray(y * w, (y + 1) * w), radius);+ blurLine(w);+ rows.set(blurred.subarray(0, w), y * w);+ }+ const out = new Float32Array(w * h);+ for (let x = 0; x < w; x++) {+ for (let y = 0; y < h; y++) line[radius + y] = rows[y * w + x]!;+ blurLine(h);+ for (let y = 0; y < h; y++) out[y * w + x] = blurred[y]!;+ }+ return out;+}++/** A normalized Gaussian kernel of `sigma`, out to three of it; a single tap at 0. */+function gaussian(sigma: number): { radius: number; weights: Float32Array } {+ if (sigma <= 0) return { radius: 0, weights: Float32Array.of(1) };+ const radius = Math.ceil(sigma * 3);+ const weights = Array.from({ length: 2 * radius + 1 }, (_, i) => Math.exp(-((i - radius) ** 2) / (2 * sigma ** 2)));+ const sum = weights.reduce((total, weight) => total + weight, 0);+ return { radius, weights: Float32Array.from(weights, (weight) => weight / sum) };+}++/**+ * Relaxes a loop `passes` times and draws it as a quadratic B-spline: from+ * the midpoint of each side to the next, bent toward the corner between. A+ * pass moves each point halfway to the middle of its neighbors; a fraction of+ * a pass moves it that much of the way, so the smoothing is continuous. A+ * point on the grid's border stays put and is drawn as a corner, so a mask+ * that fills the frame to its edge keeps its edge straight and its corners+ * square. Points are drawn times `scale`.+ */+function drawLoop(xs: number[], ys: number[], width: number, height: number, path: MaskOutlineSink, passes: number, scale: number): void {+ const n = xs.length;+ if (n < 3) return;++ const pinned = new Uint8Array(n);+ for (let i = 0; i < n; i++) pinned[i] = xs[i] === 0 || ys[i] === 0 || xs[i] === width || ys[i] === height ? 1 : 0;++ let px = Float64Array.from(xs);+ let py = Float64Array.from(ys);+ let qx = new Float64Array(n);+ let qy = new Float64Array(n);+ for (let pass = 0; pass < passes; pass++) {+ const step = Math.min(1, passes - pass) / 2;+ for (let i = 0; i < n; i++) {+ if (pinned[i]) {+ qx[i] = px[i]!;+ qy[i] = py[i]!;+ continue;+ }+ const a = i === 0 ? n - 1 : i - 1;+ const b = i === n - 1 ? 0 : i + 1;+ qx[i] = px[i]! + step * ((px[a]! + px[b]!) / 2 - px[i]!);+ qy[i] = py[i]! + step * ((py[a]! + py[b]!) / 2 - py[i]!);+ }+ [px, qx] = [qx, px];+ [py, qy] = [qy, py];+ }++ // Where the loop passes from one side to the next: halfway, or round the+ // grid's corner when the two are on borders that meet there.+ const join = (i: number, b: number): [number, number] => {+ if (pinned[i] && pinned[b] && onSide(xs[i]!, width) !== onSide(xs[b]!, width)) {+ return onSide(xs[i]!, width) ? [px[i]!, py[b]!] : [px[b]!, py[i]!];+ }+ return [(px[i]! + px[b]!) / 2, (py[i]! + py[b]!) / 2];+ };++ const [startX, startY] = join(n - 1, 0);+ path.moveTo(startX * scale, startY * scale);+ for (let i = 0; i < n; i++) {+ const [x, y] = join(i, i === n - 1 ? 0 : i + 1);+ if (pinned[i]) {+ path.lineTo(px[i]! * scale, py[i]! * scale);+ path.lineTo(x * scale, y * scale);+ } else {+ path.quadraticCurveTo(px[i]! * scale, py[i]! * scale, x * scale, y * scale);+ }+ }+ path.closePath();+}++/** Whether a point on the grid's border at `x` is on its left or right, rather than its top or bottom. */+function onSide(x: number, width: number): boolean {+ return x === 0 || x === width;+}++// A cell's corners clockwise from its top-left center, with their+// marching-squares index bits, and where its sides are crossed: side s runs+// from corner s to corner s + 1.+const CORNERS: readonly (readonly [number, number, number])[] = [[0, 0, 8], [1, 0, 4], [1, 1, 2], [0, 1, 1]];+const SIDES: readonly (readonly [number, number])[] = [[0.5, 0], [1, 0.5], [0.5, 1], [0, 0.5]];++/** The sides each cell's contour joins; the saddles (5, 10) join the object across the cell. */+const UNORIENTED: readonly (readonly number[])[] = [+ [], [3, 2], [2, 1], [3, 1], [0, 1], [3, 0, 2, 1], [0, 2], [3, 0],+ [3, 0], [0, 2], [0, 1, 3, 2], [0, 1], [3, 1], [2, 1], [3, 2], [],+];++/**+ * The same segments, each from the side it enters by to the side it leaves+ * by, the object always on the same hand.+ */+const SEGMENTS: readonly (readonly number[])[] = UNORIENTED.map((segments, index) => orient(segments, index));++/** The saddles split instead, each inside corner cut off on its own: the other saddle's sides, oriented for this one. */+const SPLIT_SEGMENTS: readonly (readonly number[])[] = SEGMENTS.map((segments, index) =>+ index === 5 || index === 10 ? orient(UNORIENTED[15 - index]!, index) : segments,+);++/**+ * Orients each segment of cell `index` by a corner it separates: the corner+ * two adjacent sides share, which is cut off from the rest, or, across the+ * cell, any corner, since the segment halves the cell.+ */+function orient(segments: readonly number[], index: number): number[] {+ const oriented: number[] = [];+ for (let i = 0; i < segments.length; i += 2) {+ const a = segments[i]!;+ const b = segments[i + 1]!;+ const shared = (a + 1) % 4 === b ? b : (b + 1) % 4 === a ? a : 0;+ const [cx, cy, bit] = CORNERS[shared]!;+ const [ax, ay] = SIDES[a]!;+ const [bx, by] = SIDES[b]!;+ // Negative when the corner is on the object's hand of a to b.+ const cross = (bx - ax) * (cy - ay) - (by - ay) * (cx - ax);+ const objectHand = (index & bit) !== 0 ? cross < 0 : cross > 0;+ oriented.push(...(objectHand ? [a, b] : [b, a]));+ }+ return oriented;+}diff --git a/packages/assets/src/mask.ts b/packages/assets/src/mask.tsnew file mode 100644index 00000000..b2826cfc--- /dev/null+++ b/packages/assets/src/mask.ts@@ -0,0 +1,318 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The mask file: a tracked matte's frames in one file, with the recipe that+// made them. A frame is a field — the model's logits on its own grid, one+// signed byte a cell (see `quantizeLogit`) — so the file keeps where the+// model put the edge between cells, not only which cells are in. The object+// is where the field is above zero. Most of a frame is saturated, far in or+// far out, and coded as runs; only the cells near the edge are written out.+// A minute of a person-sized object is a few megabytes, written once.+// The grid is stretched over the footage's frame (`width` by `height`), which+// is what a renderer fits it into; tracing its edge is the renderer's (see+// `traceMask`).+//+// The recipe (see `MaskRecipe`) is how the frames can be made again: the+// footage, the prompt, the span. The library keeps it in the manifest as well,+// so a mask whose file went missing still knows how it was made.+//+// Layout, little-endian:+//+// 0 'DSMK' magic+// 4 u32 version+// 8 u32 n byte length of the header+// 12 n bytes header: UTF-8 JSON, a `MaskHeader`+// 12 + n table per frame: u32 offset (from the data), u32 length,+// f32 score, f32 iou+// … data the frames' runs, back to back+//+// A frame covers its grid row-major in runs, each an unsigned LEB128 varint+// of its length times four plus its kind: 0 a run of cells saturated out+// (-127), 1 saturated in (+127), 2 that many cells written out after it, a+// signed byte each. A frame of length 0 has no mask (it was not tracked).++export const MASK_MIME_TYPE = 'application/x-diffusionstudio-mask';+export const MASK_EXTENSION = 'mask';++const MAGIC = [0x44, 0x53, 0x4d, 0x4b]; // 'DSMK'+const VERSION = 2;+const PREAMBLE_BYTES = 12;+const TABLE_ENTRY_BYTES = 16;++/** A field cell saturated in; its negation, saturated out. */+export const MASK_FIELD_MAX = 127;++/**+ * The logit a field byte's tanh is scaled by: a byte resolves the edge to a+ * hundredth of a cell where the model's logits cross zero, and saturates a+ * cell or two away, where the model is sure.+ */+const LOGIT_SCALE = 2;++/**+ * A logit as a field byte: `127 tanh(logit / 2)`, fine near zero and+ * saturating far from it. The sign always survives, so the cells that are in+ * are the model's exactly; 0 is out.+ */+export function quantizeLogit(logit: number): number {+ const q = Math.round(MASK_FIELD_MAX * Math.tanh(logit / LOGIT_SCALE));+ return logit > 0 ? Math.max(q, 1) : Math.min(q, 0);+}++/**+ * The logit each field byte stands for, indexed by the byte plus 127. A byte+ * of 0 is a hair below zero, so the edge between it and a byte of 1 falls+ * next to the 1, as the logits it came from had it.+ */+export const FIELD_LOGITS: Float32Array = Float32Array.from({ length: 2 * MASK_FIELD_MAX + 1 }, (_, i) => {+ const q = i - MASK_FIELD_MAX;+ if (q === 0) return -0.004;+ return LOGIT_SCALE * Math.atanh(Math.max(-0.9999, Math.min(0.9999, q / MASK_FIELD_MAX)));+});++/** A field of `cells` with nothing in it. */+export function emptyField(cells: number): Int8Array {+ return new Int8Array(cells).fill(-MASK_FIELD_MAX);+}++/** A prompt on the footage, in 0..1 of its frame; label 1 marks the object, 0 marks background. */+export type MaskPoint = { x: number; y: number; label: 0 | 1 };++/**+ * A brush stroke over the prompted frame's mask: its points in 0..1 of the+ * frame, its radius in 0..1 of the frame's height; label 1 adds, 0 erases.+ */+export type MaskStroke = { label: 0 | 1; radius: number; points: { x: number; y: number }[] };++/**+ * How a mask's frames were made, enough to make them again: the object+ * prompted by `points` on `seedFrame` of the footage, its mask there+ * corrected by `strokes`, followed through every frame from `first` on — one+ * a frame of the file, at the file's rate.+ */+export interface MaskRecipe {+ /** The model that made the frames. */+ model: string;+ /** The content id of the footage the object was tracked on. */+ source: string;+ /** The source frame, at the file's rate, frame 0 of the file belongs to. */+ first: number;+ /** The source frame, at the file's rate, the points were placed on. */+ seedFrame: number;+ points: MaskPoint[];+ /** Painted over the model's mask of the seed frame, in order. */+ strokes?: MaskStroke[];+}++export interface MaskHeader {+ /** The field's grid: what every frame holds. */+ gridWidth: number;+ gridHeight: number;+ /** The frame the grid covers: the footage's, which the grid is stretched over. */+ width: number;+ height: number;+ frameRate: number;+ frameCount: number;+ recipe?: MaskRecipe;+}++/** One frame's segmentation: its field row-major, a byte a cell (see `quantizeLogit`); in where above zero. */+export type MaskFrame = {+ field: Int8Array;+ /** Whether the object is in frame: at or below zero the model saw it occluded. */+ score: number;+ /** The model's own estimate of the mask's quality, 0 to 1. */+ iou: number;+};++/** Whether `bytes` begin like a mask file. */+export function isMaskFileHead(bytes: Uint8Array): boolean {+ return MAGIC.every((byte, index) => bytes[index] === byte);+}++/** Writes a mask file: `frames[i]` is frame `i`, null where there is no mask. */+export function encodeMaskFile(header: Omit<MaskHeader, 'frameCount'>, frames: readonly (MaskFrame | null)[]): Blob {+ const full: MaskHeader = { ...header, frameCount: frames.length };+ const json = new TextEncoder().encode(JSON.stringify(full));+ const cells = header.gridWidth * header.gridHeight;++ const table = new DataView(new ArrayBuffer(frames.length * TABLE_ENTRY_BYTES));+ const runs = new RunWriter();+ for (let i = 0; i < frames.length; i++) {+ const frame = frames[i];+ const at = i * TABLE_ENTRY_BYTES;+ const offset = runs.length;+ if (frame) encodeRuns(frame.field, cells, runs);+ table.setUint32(at, offset, true);+ table.setUint32(at + 4, runs.length - offset, true);+ table.setFloat32(at + 8, frame?.score ?? 0, true);+ table.setFloat32(at + 12, frame?.iou ?? 0, true);+ }++ const preamble = new DataView(new ArrayBuffer(PREAMBLE_BYTES));+ MAGIC.forEach((byte, index) => preamble.setUint8(index, byte));+ preamble.setUint32(4, VERSION, true);+ preamble.setUint32(8, json.byteLength, true);++ return new Blob([preamble, json, table, runs.written()], { type: MASK_MIME_TYPE });+}++/** The header of a mask file, read without the frames. */+export async function readMaskHeader(blob: Blob): Promise<MaskHeader> {+ const preamble = new Uint8Array(await blob.slice(0, PREAMBLE_BYTES).arrayBuffer());+ const length = readPreamble(preamble);+ const json = await blob.slice(PREAMBLE_BYTES, PREAMBLE_BYTES + length).arrayBuffer();+ return parseHeader(json);+}++/** A mask file in memory, its frames decoded one at a time as they are asked for. */+export class MaskFile {+ public readonly header: MaskHeader;+ private readonly table: DataView;+ private readonly data: Uint8Array;++ private constructor(header: MaskHeader, table: DataView, data: Uint8Array) {+ this.header = header;+ this.table = table;+ this.data = data;+ }++ public static async read(blob: Blob): Promise<MaskFile> {+ const bytes = new Uint8Array(await blob.arrayBuffer());+ const length = readPreamble(bytes);+ const header = parseHeader(bytes.subarray(PREAMBLE_BYTES, PREAMBLE_BYTES + length));+ const tableStart = PREAMBLE_BYTES + length;+ const dataStart = tableStart + header.frameCount * TABLE_ENTRY_BYTES;+ if (bytes.byteLength < dataStart) throw new Error('The mask file is cut short');+ const table = new DataView(bytes.buffer, bytes.byteOffset + tableStart, dataStart - tableStart);+ return new MaskFile(header, table, bytes.subarray(dataStart));+ }++ public get frameCount(): number {+ return this.header.frameCount;+ }++ /** Whether frame `index` has a mask: in range, and tracked. */+ public has(index: number): boolean {+ return index >= 0 && index < this.frameCount && this.table.getUint32(index * TABLE_ENTRY_BYTES + 4, true) > 0;+ }++ public score(index: number): number {+ return this.table.getFloat32(index * TABLE_ENTRY_BYTES + 8, true);+ }++ public iou(index: number): number {+ return this.table.getFloat32(index * TABLE_ENTRY_BYTES + 12, true);+ }++ /**+ * Decodes frame `index`'s field into `field`, a cell a byte. False, and+ * nothing written, when the frame has no mask.+ */+ public field(index: number, field: Int8Array): boolean {+ if (!this.has(index)) return false;+ const at = index * TABLE_ENTRY_BYTES;+ const offset = this.table.getUint32(at, true);+ const length = this.table.getUint32(at + 4, true);+ const size = Math.min(field.length, this.header.gridWidth * this.header.gridHeight);++ let cursor = offset;+ const end = offset + length;+ let position = 0;+ while (cursor < end && position < size) {+ let value = 0;+ let shift = 0;+ let byte: number;+ do {+ byte = this.data[cursor++]!;+ value += (byte & 0x7f) * 2 ** shift;+ shift += 7;+ } while (byte & 0x80 && cursor < end);+ const kind = value % 4;+ const stop = Math.min(size, position + Math.floor(value / 4));+ if (kind === RUN_LITERAL) {+ for (; position < stop && cursor < end; position++) field[position] = (this.data[cursor++]! << 24) >> 24;+ } else {+ field.fill(kind === RUN_IN ? MASK_FIELD_MAX : -MASK_FIELD_MAX, position, stop);+ }+ position = stop;+ }+ if (position < size) field.fill(-MASK_FIELD_MAX, position, size);+ return true;+ }+}++/** The header length a preamble gives, after checking it is one. */+function readPreamble(bytes: Uint8Array): number {+ if (bytes.byteLength < PREAMBLE_BYTES || !isMaskFileHead(bytes)) throw new Error('Not a mask file');+ const view = new DataView(bytes.buffer, bytes.byteOffset, PREAMBLE_BYTES);+ const version = view.getUint32(4, true);+ if (version !== VERSION) throw new Error(`Unsupported mask file version: ${version}`);+ return view.getUint32(8, true);+}++function parseHeader(json: ArrayBuffer | Uint8Array): MaskHeader {+ const header = JSON.parse(new TextDecoder().decode(json)) as MaskHeader;+ const counts = [header.gridWidth, header.gridHeight, header.width, header.height, header.frameCount];+ if (!counts.every((value) => Number.isInteger(value) && value >= 0) || !(header.frameRate > 0)) {+ throw new Error('The mask file header is malformed');+ }+ return header;+}++const RUN_OUT = 0;+const RUN_IN = 1;+const RUN_LITERAL = 2;++/** Appends the runs of the first `cells` of `field` to `out`: saturated stretches as runs, the rest written out. */+function encodeRuns(field: Int8Array, cells: number, out: RunWriter): void {+ for (let i = 0; i < cells;) {+ const value = field[i]!;+ let end = i + 1;+ if (value === MASK_FIELD_MAX || value === -MASK_FIELD_MAX) {+ while (end < cells && field[end] === value) end++;+ out.varint((end - i) * 4 + (value > 0 ? RUN_IN : RUN_OUT));+ } else {+ while (end < cells && Math.abs(field[end]!) !== MASK_FIELD_MAX) end++;+ out.varint((end - i) * 4 + RUN_LITERAL);+ out.bytes(field.subarray(i, end));+ }+ i = end;+ }+}++/** A growing byte buffer of varints. */+class RunWriter {+ private buffer: Uint8Array<ArrayBuffer> = new Uint8Array(64 * 1024);+ public length = 0;++ public varint(value: number): void {+ this.reserve(5);+ while (value >= 0x80) {+ this.buffer[this.length++] = (value & 0x7f) | 0x80;+ value = Math.floor(value / 0x80);+ }+ this.buffer[this.length++] = value;+ }++ /** Appends `values` as they are, a signed byte each. */+ public bytes(values: Int8Array): void {+ this.reserve(values.length);+ this.buffer.set(new Uint8Array(values.buffer, values.byteOffset, values.length), this.length);+ this.length += values.length;+ }++ public written(): Uint8Array<ArrayBuffer> {+ return this.buffer.subarray(0, this.length);+ }++ private reserve(bytes: number): void {+ if (this.length + bytes <= this.buffer.length) return;+ let size = this.buffer.length * 2;+ while (size < this.length + bytes) size *= 2;+ const grown = new Uint8Array(size);+ grown.set(this.buffer.subarray(0, this.length));+ this.buffer = grown;+ }+}diff --git a/packages/assets/src/probe.ts b/packages/assets/src/probe.tsindex 1f832b4b..df9bde46 100644--- a/packages/assets/src/probe.ts+++ b/packages/assets/src/probe.ts@@ -7,7 +7,10 @@import { ALL_FORMATS, BlobSource, Input, UrlSource } from 'mediabunny';+import { isMaskFileHead, MASK_MIME_TYPE, readMaskHeader } from './mask';+import type { InputTrack } from 'mediabunny';+import type { MaskRecipe } from './mask';import type { Asset } from './types';export const DEFAULT_SEQUENCE_FPS = 30;@@ -18,7 +21,8 @@ export type ProbeResult = Extract<| { type: 'AUDIO'; duration: number; sampleRate: number; channels: number }| { type: 'VIDEO'; duration: number; width: number; height: number; frameRate: number; bitRate: number; sampleRate?: number; channels?: number }| { type: 'TRANSCRIPT' }- | { type: 'SCRIPT' },+ | { type: 'SCRIPT' }+ | { type: 'MASK'; width: number; height: number; frameRate: number; duration: number; recipe?: MaskRecipe },{ type: Asset['type'] }>;@@ -28,9 +32,9 @@ const TRANSCRIPT_TYPES = new Set(['application/json', 'application/x-subrip', 't* The MIME type of a file, or of the resource behind a URL. Subtitle files go* by extension (OS registries rarely map .srt); images fall back to their* magic bytes when nothing declared a type (a download filed without an- * extension has none to go by); audio and video are sniffed by mediabunny,- * since a container's declared type is often wrong or empty. Null when the- * file is not something the library takes.+ * extension has none to go by), and so do mask files; audio and video are+ * sniffed by mediabunny, since a container's declared type is often wrong or+ * empty. Null when the file is not something the library takes.*/export async function detectMimeType(input: Blob | string): Promise<string | null> {const name = typeof input === 'string' ? input.split(/[?#]/)[0]! : (input as File).name ?? '';@@ -39,6 +43,7 @@ export async function detectMimeType(input: Blob | string): Promise<string | nulif (/\.srt$/i.test(name) || mimeType === 'application/x-subrip') return 'application/x-subrip';if (/\.vtt$/i.test(name) || mimeType === 'text/vtt') return 'text/vtt';if (/\.json$/i.test(name) || mimeType === 'application/json') return 'application/json';+ if (mimeType === MASK_MIME_TYPE) return MASK_MIME_TYPE;if (mimeType?.startsWith('image/')) return mimeType;if (mimeType?.startsWith('text/html')) return mimeType;@@ -46,6 +51,7 @@ export async function detectMimeType(input: Blob | string): Promise<string | nulif (typeof input !== 'string') {const sniffed = await sniffImageType(input);if (sniffed) return sniffed;+ if (await sniffMaskFile(input)) return MASK_MIME_TYPE;}try {@@ -59,6 +65,15 @@ export async function detectMimeType(input: Blob | string): Promise<string | nulreturn null;}+/** Whether a blob is a mask file (see `mask.ts`), by its magic. */+async function sniffMaskFile(input: Blob): Promise<boolean> {+ try {+ return isMaskFileHead(new Uint8Array(await input.slice(0, 4).arrayBuffer()));+ } catch {+ return false;+ }+}+/** Image container brands that ride in an ISO base media `ftyp` box. */const IMAGE_BRANDS: Record<string, string> = {avif: 'image/avif',@@ -127,6 +142,11 @@ export async function probeMedia(file: Blob, mimeType: string): Promise<ProbeResif (TRANSCRIPT_TYPES.has(mimeType)) return { type: 'TRANSCRIPT' };+ if (mimeType === MASK_MIME_TYPE) {+ const { width, height, frameRate, frameCount, recipe } = await readMaskHeader(file);+ return { type: 'MASK', width, height, frameRate, duration: frameCount / frameRate, ...(recipe ? { recipe } : {}) };+ }+if (mimeType.startsWith('audio/') || mimeType.startsWith('video/')) {const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });const audioTrack = await input.getPrimaryAudioTrack();diff --git a/packages/assets/src/types.ts b/packages/assets/src/types.tsindex 62f754d8..e139256f 100644--- a/packages/assets/src/types.ts+++ b/packages/assets/src/types.ts@@ -2,6 +2,8 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */+import type { MaskRecipe } from './mask';+/** Whatever hands out the asset's bytes: a real File on desktop, an OPFS* file on the web, a fetched blob for a URL. */export interface AssetFileHandle {@@ -106,13 +108,28 @@ export interface SequenceAsset extends AssetBase {directoryHandle: AssetDirectoryHandle;}+/**+ * A tracked matte in a mask file (see `mask.ts`): the model's logits on its+ * grid, a field a frame, stretched over the footage's frame. `recipe` is how the frames were made,+ * kept here too so a mask whose file went missing can be made again.+ */+export interface MaskAsset extends AssetBase {+ type: 'MASK';+ width: number;+ height: number;+ frameRate: number;+ duration: number;+ recipe?: MaskRecipe;+}+export type Asset =| ImageAsset| AudioAsset| VideoAsset| TranscriptAsset| ScriptAsset- | SequenceAsset;+ | SequenceAsset+ | MaskAsset;export type AssetType = Asset['type'];diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 16381a62..370657a5 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -630,6 +630,13 @@ export type MaskProps = Pick<CompositeProps, "hidden"> & TrackChildren & {opacity?: number;/** Covers what the picture does not instead. Default false. */inverted?: boolean;+ /**+ * How much a mask file's edge is smoothed, 0–1. At 0 it is exactly where+ * the model put it; the more, the rounder,+ * and the more of what is thin or small melts away. Sharp at any size+ * either way. Default 0.25.+ */+ smoothing?: number;};/**diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 4717d76d..cc3a15ce 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -4,6 +4,7 @@import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, Constraint, ConstraintCache, ConstraintType, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Matte, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, setCameraMatrix, setPlayhead, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';+import { DEFAULT_MASK_SMOOTHING } from '@diffusionstudio/assets';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import { createSignal } from 'solid-js';import { SVGElements } from 'solid-js/web';@@ -974,6 +975,12 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {entity.set(Matte, { inverted: value === true });return;}+ case 'smoothing': {+ if (!entity.has(Matte)) return;+ const amount = toNumber(value) ?? DEFAULT_MASK_SMOOTHING;+ entity.set(Matte, { smoothing: Math.min(1, Math.max(0, amount)) });+ return;+ }case 'keepAspectRatio': {this.syncAspectLock(node);return;diff --git a/packages/reconciler/tsconfig.json b/packages/reconciler/tsconfig.jsonindex 516eda7b..50672c68 100644--- a/packages/reconciler/tsconfig.json+++ b/packages/reconciler/tsconfig.json@@ -4,7 +4,7 @@"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo","erasableSyntaxOnly": false,"lib": ["es2024", "DOM", "DOM.Iterable"],- "types": ["@webgpu/types"]+ "types": ["@webgpu/types", "@types/wicg-file-system-access"]},"include": ["src"]}diff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.tsindex e0a88cc5..662716a9 100644--- a/packages/runtime/src/actions/assets.ts+++ b/packages/runtime/src/actions/assets.ts@@ -10,7 +10,7 @@ import { Ai, AssetId, Library, Paint, SourceFrameRate } from '../traits';import { PaintType } from '../constants';import type { Entity, World } from 'koota';-import type { Asset, AssetLibrary, SequenceAsset } from '@diffusionstudio/assets';+import type { Asset, AssetLibrary, MaskAsset, SequenceAsset } from '@diffusionstudio/assets';import type { GenAi } from '../ai';/** The world's asset library; throws when the host attached none. */@@ -30,8 +30,8 @@ export function getAi(world: World): GenAi {/*** Binds an entity to an asset: stamps its AssetId, and follows the asset with* the paint. A media paint is whichever of the two the asset can be drawn- * through — a frames directory decodes and draws exactly as a video does, so- * it paints as one — and it is re-decided on every bind, so an element handed+ * through — a frames directory or a mask file decodes and draws exactly as a+ * video does, so it paints as one — and it is re-decided on every bind, so an element handed* a different kind of source shows it rather than nothing at all. Paints that* are not media (a gradient, a shader) are left alone: their fill is not the* asset.@@ -44,7 +44,7 @@ export function bindAsset(entity: Entity, asset: Asset): void {if (paint !== PaintType.IMAGE && paint !== PaintType.VIDEO) return;const wanted = asset.type === 'IMAGE' ? PaintType.IMAGE- : asset.type === 'VIDEO' || asset.type === 'SEQUENCE' ? PaintType.VIDEO+ : asset.type === 'VIDEO' || asset.type === 'SEQUENCE' || asset.type === 'MASK' ? PaintType.VIDEO// Audio and transcripts have nothing to draw; leave the paint as authored.: paint;@@ -52,11 +52,11 @@ export function bindAsset(entity: Entity, asset: Asset): void {}/**- * The rate a frames directory is played at: the element's own `frameRate`- * when it sets one, else the rate the library gave the asset. `ignoreAuthored`- * reads the element's as absent, for a handler of its removal.+ * The rate a frames directory or mask file is played at: the element's own+ * `frameRate` when it sets one, else the rate the library gave the asset.+ * `ignoreAuthored` reads the element's as absent, for a handler of its removal.*/-export function getSequenceFrameRate(entity: Entity, asset: SequenceAsset, ignoreAuthored = false): number {+export function getSequenceFrameRate(entity: Entity, asset: SequenceAsset | MaskAsset, ignoreAuthored = false): number {const authored = ignoreAuthored ? 0 : entity.get(SourceFrameRate)?.value ?? 0;return authored > 0 ? authored : asset.frameRate;}@@ -68,7 +68,7 @@ export function getSequenceFrameRate(entity: Entity, asset: SequenceAsset, ignor* is the element's to set.*/export function getSourceDuration(entity: Entity, asset: Asset, ignoreAuthoredRate = false): number | null {- if (asset.type === 'SEQUENCE') {+ if (asset.type === 'SEQUENCE' || asset.type === 'MASK') {const frames = asset.duration * asset.frameRate;return frames / getSequenceFrameRate(entity, asset, ignoreAuthoredRate);}diff --git a/packages/runtime/src/media/index.ts b/packages/runtime/src/media/index.tsindex b30f5a43..ae1a75bf 100644--- a/packages/runtime/src/media/index.ts+++ b/packages/runtime/src/media/index.ts@@ -13,6 +13,7 @@ export * from './handoff';export * from './html';export * from './image';export * from './keyframe-index';+export * from './mask';export * from './sequence';export * from './shader';export * from './time-stretcher';diff --git a/packages/runtime/src/media/mask.ts b/packages/runtime/src/media/mask.tsnew file mode 100644index 00000000..d1601992--- /dev/null+++ b/packages/runtime/src/media/mask.ts@@ -0,0 +1,145 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { DEFAULT_MASK_SMOOTHING, MASK_FIELD_MAX, MaskFile, traceMask } from '@diffusionstudio/assets';++import { getAssetFile } from '../actions/assets';++import type { AssetStat, MaskAsset } from '@diffusionstudio/assets';++/** White where the object is: a matte is read by its alpha. */+const FOREGROUND = '#ffffff';++/** A picture's shortest side, for whatever asks for one: enough that its edge is not the blur of a small bitmap. */+const PICTURE_SIDE = 512;++/**+ * A frame's edge at a smoothing, as a path over a picture `width` by+ * `height`: the canvas's, so it fits where the picture would.+ */+export type MaskOutline = { path: Path2D; width: number; height: number; smoothing: number };++/**+ * Plays a mask file (see `MaskFile`). The file is read once — a few+ * megabytes a minute at most — and a frame's field is decoded from its runs+ * when it is asked for, which costs microseconds: nothing is decoded ahead,+ * and there is no cache to keep warm. Its edge is traced into an outline at+ * the matte's smoothing (see `traceMask`) when a renderer asks for it, which+ * it fills at the size the mask lands at; for whatever asks for a picture+ * instead, the outline is filled on a canvas the shape of the footage.+ */+export class MaskDecoder {+ public errored = false;+ public disposed = false;+ public asset: MaskAsset;+ public readonly initialized: Promise<void>;++ /** Frames per second the file is played at: its own, unless the element retimes it (see SourceFrameRate). */+ public frameRate: number;++ /** The asset's stat when the file was read: a restored file lands under the same id with a new one. */+ public readonly stat: AssetStat | undefined;++ private file: MaskFile | null = null;+ private field: Int8Array | null = null;+ private drawn = -1;+ /** Whether the canvas holds the drawn frame, and its outline at the smoothing last asked for. */+ private pictured = false;+ private outline: MaskOutline | null = null;++ public readonly canvas = new OffscreenCanvas(0, 0);+ private readonly ctx = this.canvas.getContext('2d')!;++ public constructor(asset: MaskAsset) {+ this.asset = asset;+ this.stat = asset.stat;+ this.frameRate = asset.frameRate;+ this.initialized = this.initialize();+ }++ private async initialize(): Promise<void> {+ try {+ const file = await MaskFile.read(await getAssetFile(this.asset));+ const { gridWidth, gridHeight, width, height } = file.header;+ if (this.disposed || gridWidth === 0 || gridHeight === 0) return;++ this.file = file;+ this.field = new Int8Array(gridWidth * gridHeight);++ const scale = PICTURE_SIDE / Math.min(width || gridWidth, height || gridHeight);+ this.canvas.width = Math.max(1, Math.round((width || gridWidth) * scale));+ this.canvas.height = Math.max(1, Math.round((height || gridHeight) * scale));+ } catch {+ this.errored = true;+ }+ }++ /** Whether this decoder is for bytes the asset no longer has: its file went missing and was made again. */+ public isStale(asset: MaskAsset): boolean {+ return this.errored && asset.stat !== this.stat;+ }++ public seekTo(frame: number, frameRate: number): Promise<void> | void {+ const target = Math.round((frame / frameRate) * this.frameRate);+ if (this.file) return this.draw(target);+ if (this.errored || this.disposed) return;+ return this.initialized.then(() => this.draw(target));+ }++ private draw(target: number): void {+ const file = this.file;+ if (!file || !this.field || this.disposed || file.frameCount === 0) return;++ // Past either end the nearest frame stands, as a sequence's last frame does.+ const index = Math.min(Math.max(target, 0), file.frameCount - 1);+ if (index === this.drawn) return;++ // A frame the object was not tracked on masks nothing.+ if (!file.field(index, this.field)) this.field.fill(-MASK_FIELD_MAX);+ this.drawn = index;+ this.pictured = false;+ this.outline = null;+ }++ /** The drawn frame's outline at the default smoothing, filled on the canvas. */+ public toBitmap(): OffscreenCanvas | null {+ const outline = this.getOutline(DEFAULT_MASK_SMOOTHING);+ if (!outline) return null;+ if (!this.pictured) {+ const { width, height } = this.canvas;+ this.ctx.clearRect(0, 0, width, height);+ this.ctx.fillStyle = FOREGROUND;+ this.ctx.fill(outline.path);+ this.pictured = true;+ }+ return this.canvas;+ }++ /** The drawn frame's edge, traced at `smoothing`, over the canvas. */+ public getOutline(smoothing: number): MaskOutline | null {+ const file = this.file;+ if (this.drawn < 0 || !file || !this.field) return null;+ if (this.outline?.smoothing !== smoothing) {+ const { gridWidth, gridHeight } = file.header;+ const { width, height } = this.canvas;+ const traced = new Path2D();+ traceMask(this.field, gridWidth, gridHeight, traced, smoothing);+ const path = new Path2D();+ path.addPath(traced, new DOMMatrix([width / gridWidth, 0, 0, height / gridHeight, 0, 0]));+ this.outline = { path, width, height, smoothing };+ }+ return this.outline;+ }++ public dispose(): void {+ if (this.disposed) return;+ this.disposed = true;+ this.file = null;+ this.field = null;+ this.outline = null;+ this.drawn = -1;+ this.canvas.width = 0;+ this.canvas.height = 0;+ }+}diff --git a/packages/runtime/src/media/video.ts b/packages/runtime/src/media/video.tsindex 7540149c..40b0142a 100644--- a/packages/runtime/src/media/video.ts+++ b/packages/runtime/src/media/video.ts@@ -9,6 +9,7 @@ import { assert } from '../utils/assert';import { getAsset, getAssetFile, getSequenceFrameRate } from '../actions/assets';import { FrameCache } from './frame-cache';import { getKeyframeIndex } from './keyframe-index';+import { MaskDecoder } from './mask';import { SequenceDecoder } from './sequence';import type { Entity, World } from 'koota';@@ -677,13 +678,13 @@ export class VideoExporter {}/**- * What a video paint decodes through. Three implementations of one interface- * — `seekTo`, `toBitmap`, `idle`, `dispose` — picked by what the source turns- * out to be and what the world is doing with it: a demuxed buffer for playing- * a file, an exact-seeking reader for encoding one, and a frames directory- * read off disk. Nothing downstream of `resolveVideoDecoder` asks which.+ * What a video paint decodes through. Four implementations of one interface+ * — `seekTo`, `toBitmap`, `dispose` — picked by what the source turns out to+ * be and what the world is doing with it: a demuxed buffer for playing a+ * file, an exact-seeking reader for encoding one, a frames directory read off+ * disk, and a mask file. Nothing downstream of `resolveVideoDecoder` asks which.*/-export type VideoDecoderInstance = VideoBuffer | VideoExporter | SequenceDecoder;+export type VideoDecoderInstance = VideoBuffer | VideoExporter | SequenceDecoder | MaskDecoder;const videoTrackCache = new Map<string, Promise<InputVideoTrack | null>>();@@ -719,9 +720,9 @@ export function getVideoTrack(source: VideoAsset) {* The decoder `entity`'s video paint draws from, built on first use and kept* until the asset it was built for is no longer the one asked for.*- * A sequence's rate is the element's to set, so it is pushed on every call- * rather than fixed at construction — re-reading a folder to play it slower- * would be a rebuild for nothing.+ * A sequence's or mask's rate is the element's to set, so it is pushed on+ * every call rather than fixed at construction — re-reading a folder to play+ * it slower would be a rebuild for nothing.*/export function resolveVideoDecoder(world: World, entity: Entity): VideoDecoderInstance | null {const assetId = entity.get(AssetId)?.value;@@ -738,6 +739,8 @@ export function resolveVideoDecoder(world: World, entity: Entity): VideoDecoderIif (existing instanceof SequenceDecoder) {existing.hasCache = hasCache;existing.frameRate = getSequenceFrameRate(entity, existing.asset);+ } else if (existing instanceof MaskDecoder) {+ existing.frameRate = getSequenceFrameRate(entity, existing.asset);}return existing;}@@ -752,6 +755,9 @@ export function resolveVideoDecoder(world: World, entity: Entity): VideoDecoderIif (asset.type === 'SEQUENCE') {decoder = new SequenceDecoder(asset, hasCache);decoder.frameRate = getSequenceFrameRate(entity, asset);+ } else if (asset.type === 'MASK') {+ decoder = new MaskDecoder(asset);+ decoder.frameRate = getSequenceFrameRate(entity, asset);} else if (asset.type === 'VIDEO') {decoder = hasCache ? new VideoBuffer(asset) : new VideoExporter(asset);} else {diff --git a/packages/runtime/src/systems/playback.ts b/packages/runtime/src/systems/playback.tsindex ebf52506..252b6197 100644--- a/packages/runtime/src/systems/playback.ts+++ b/packages/runtime/src/systems/playback.ts@@ -12,7 +12,7 @@ import { Not, Or } from 'koota';import { store } from '../world/store';import { PaintType } from '../constants';import {- ChildOf, Hidden, Culled, Dragging,+ ChildOf, Hidden, Culled, Dragging, AssetId, Cache, Matte,Geometry, Group, AdjustmentLayer, Paint, Audio, Caption, Muted, Soloed,Sequential, Transition, Playback, Workarea,AudioPlayback, Computed,@@ -221,6 +221,49 @@ function forwardImageDecoder(world: World, _scene: Entity, _entity: Entity, fill}}+/**+ * Seeks the frame sequences of the mattes under the entity's effects: a+ * matte's frame `i` belongs to source frame `offset + i` of the clip, so the+ * seek is the clip's own source frame less the offset, at the rate the+ * sequence was written at (the composition's, authored as its frameRate).+ */+function forwardMatteDecoders(world: World, scene: Entity, entity: Entity): void {+ const cache = store(world, Cache);+ const effects = cache.effects[entity.id()];+ if (!effects?.length) return;++ const computed = store(world, Computed);+ const matteStore = store(world, Matte);+ const eid = entity.id();+ const fps = world.get(FrameRate)?.value ?? 30;++ const globalFrame = computed.localTime[scene.id()]!;+ const localFrame = computed.localTime[eid]!;+ const start = computed.start[eid]!;+ const end = computed.end[eid]!;+ const hasCache = world.get(Mode)?.value === 'realtime';+ const warmupDecoder = globalFrame >= start - WARMUP_FRAMES && globalFrame < end + WARMUP_FRAMES && hasCache;+ if (computed.visibility[eid] !== 1 && !warmupDecoder) return;++ const source = getSourceWindow(entity);+ const sourceFrame = clamp(localFrame, source.in, source.out);++ for (const effect of effects) {+ if (effect.has(Hidden)) continue;+ for (const matte of cache.mattes[effect.id()] ?? []) {+ if (matte.has(Hidden) || !matte.has(AssetId)) continue;++ const decoder = resolveVideoDecoder(world, matte);+ if (!decoder) continue;+ const frame = Math.max(0, sourceFrame - (matteStore.offset[matte.id()] ?? 0));+ // Seek first: the editor keeps no promise list, and an optional call+ // skips its arguments along with itself.+ const seek = decoder.seekTo(frame, fps);+ framePromises(world)?.push(seek ?? null);+ }+ }+}+/*** Forward decoders for a child entity and its node descendants.*/@@ -269,6 +312,10 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {if (ready) framePromises(world)?.push(ready);}}++ if (visualsEnabled && entity.has(Geometry)) {+ forwardMatteDecoders(world, scene, entity);+ }}if (entity.has(Caption) && visualsEnabled) {diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex 6a39601d..4a62d73a 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -8,6 +8,7 @@// regions are pushed callback-less (see HitRegions); the app's input layer// attaches its handlers.+import { DEFAULT_MASK_SMOOTHING } from '@diffusionstudio/assets';import { Not, Or } from 'koota';import { store } from '../world/store';@@ -16,9 +17,9 @@ import {TransitionType,} from '../constants';import {- ChildOf, Hidden, Culled, Interactive, IsMask,+ ChildOf, Hidden, Culled, Interactive, IsMask, AssetId,ClipsContent, Geometry, Group, Paint, Color, Caption, ScaleMode, Shader,- BlendMode, Effect, Transition, MixedCornerRadius,+ BlendMode, Effect, Matte, Transition, MixedCornerRadius,LocalTransform, WorldTransform, Computed, Cache,Host,Mode, FrameRate, Camera, Background, RenderSurface,@@ -32,10 +33,11 @@ import { FAILED_COLOR, getGeneratingColor, getSourceFailure, isGenerating } fromimport { applyStrokeStyle } from '../utils/stroke';import { renderText } from '../utils/text';import { getTransitionWindow } from '../utils/transition';-import { getIntrinsicPaint } from '../utils/time';+import { drawOnto, getSurfaceContext } from '../utils/surface';+import { findGeometryAssetSource, getIntrinsicPaint } from '../utils/time';import { createLinearGradient, createRadialGradient } from './gradients';import {- resolveImageDecoder, resolveVideoDecoder,+ MaskDecoder, resolveImageDecoder, resolveVideoDecoder,resolveCaptionDecoder, resolveShaderHost, resolveWaveformPeaks,} from '../media';@@ -47,7 +49,7 @@ type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;const MISSING_ASSET_COLOR = '#5C2828';function getCtx(world: World): Ctx2D {- return world.get(RenderSurface)!.ctx!;+ return getSurfaceContext(world)!;}export function drawRectPath(world: World, entity: Entity): void {@@ -103,7 +105,7 @@ export function drawRectPath(world: World, entity: Entity): void {ctx.closePath();}-function getScaledImageProps(+export function getScaledImageProps(mode: number,imgW: number,imgH: number,@@ -169,6 +171,10 @@ function effectFilter(world: World, sub: Entity): string | null {const clamped = Math.min(1, Math.max(0, value));return clamped > EPSILON ? `sepia(${clamped})` : null;}+ if (type === EffectType.OPACITY) {+ const clamped = Math.min(1, Math.max(0, value));+ return 1 - clamped > EPSILON ? `opacity(${clamped})` : null;+ }return null;}@@ -290,6 +296,16 @@ function renderHtmlFill(world: World, entity: Entity, source: Entity): void {ctx.restore();}+/** The frame `source`'s decoder has for a box of `w` by `h`, and whether the decoder gave up. */+function resolveMediaFrame(world: World, source: Entity, kind: MediaKind, w: number, h: number): { frame: MediaFrame | null; failed: boolean } {+ if (kind === 'IMAGE') {+ const decoder = resolveImageDecoder(world, source)?.decoder;+ return { frame: decoder?.getBitmap(w, h) ?? null, failed: decoder?.failed ?? false };+ }+ const decoder = resolveVideoDecoder(world, source);+ return { frame: decoder?.toBitmap() ?? null, failed: decoder?.errored ?? false };+}+/*** Draws the current frame of `source` (an image or video paint, or the* geometry itself for intrinsic media) into `entity`'s box, fitted by the@@ -302,17 +318,7 @@ function renderMedia(world: World, entity: Entity, source: Entity, kind: MediaKiconst w = computed.width[eid]!;const h = computed.height[eid]!;- let frame: MediaFrame | null | undefined;- let failed = false;- if (kind === 'IMAGE') {- const decoder = resolveImageDecoder(world, source)?.decoder;- frame = decoder?.getBitmap(w, h);- failed = decoder?.failed ?? false;- } else {- const decoder = resolveVideoDecoder(world, source);- frame = decoder?.toBitmap();- failed = decoder?.errored ?? false;- }+ const { frame, failed } = resolveMediaFrame(world, source, kind, w, h);if (frame) {ctx.save();@@ -793,30 +799,57 @@ export function renderNode(world: World, entity: Entity): void {local.f[eid]!,);- const worldTransform = store(world, WorldTransform);for (const mask of store(world, Cache).masks[eid] ?? []) {if (computed.visibility[mask.id()] === 0) continue;- ctx.save();- ctx.setTransform(- worldTransform.a[mask.id()]!,- worldTransform.b[mask.id()]!,- worldTransform.c[mask.id()]!,- worldTransform.d[mask.id()]!,- worldTransform.e[mask.id()]!,- worldTransform.f[mask.id()]!,- );- drawRectPath(world, mask);- ctx.restore();- ctx.clip();+ clipToMask(world, mask);}+ // An effect limited by a `<mask>` takes the node through layers.+ const passes = effectPasses(world, entity);+// Opacity and blend mode. The store slot may hold a destroyed entity's// value (ids are recycled), so it is only readable behind has().ctx.globalAlpha *= computed.opacity[eid]!;const bi = entity.has(BlendMode) ? store(world, BlendMode).value[eid] ?? 0 : 0;if (bi !== 0) ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[bi]!;- const effects = buildEffects(world, entity);+ if (passes) {+ renderLayered(world, entity, passes);+ } else {+ renderContent(world, entity, buildEffects(world, entity));+ }++ ctx.restore();+}++/** Clips the current context to the box of `mask`, in world space. */+function clipToMask(world: World, mask: Entity): void {+ const ctx = getCtx(world);+ ctx.save();+ setWorldTransform(world, ctx, mask);+ drawRectPath(world, mask);+ ctx.restore();+ ctx.clip();+}++function setWorldTransform(world: World, ctx: Ctx2D, entity: Entity): void {+ const worldTransform = store(world, WorldTransform);+ const eid = entity.id();+ ctx.setTransform(+ worldTransform.a[eid]!,+ worldTransform.b[eid]!,+ worldTransform.c[eid]!,+ worldTransform.d[eid]!,+ worldTransform.e[eid]!,+ worldTransform.f[eid]!,+ );+}++/** The node's own pixels and its children on the current context, under `filter` when one is given. */+function renderContent(world: World, entity: Entity, effects: string | null): void {+ const ctx = getCtx(world);+ const eid = entity.id();+let initialFilter = 'none';if (effects !== null) {@@ -824,7 +857,6 @@ export function renderNode(world: World, entity: Entity): void {ctx.filter = effects;}-if (entity.has(Caption)) {renderCaptionNode(world, entity);} else if (store(world, Geometry).value[eid] === GeometryType.TEXT) {@@ -860,8 +892,243 @@ export function renderNode(world: World, entity: Entity): void {if (initialFilter !== 'none') {ctx.filter = initialFilter;}+}++// ── Effect passes ────────────────────────────────────────────++/** One step of a node's filter pipeline: a run of plain filters, or one effect limited by its mattes. */+type EffectPass =+ | { kind: 'filter'; filter: string }+ | { kind: 'masked'; filter: string | null; mattes: Entity[]; opacity: boolean };++/**+ * The node's effects as passes, or null when none is masked and one CSS+ * filter string (see `buildEffects`) does the whole job in a single draw.+ * The node's own blur leads, and consecutive unmasked effects fold into one+ * filter run, so a node pays a layer per masked effect and nothing more.+ */+function effectPasses(world: World, entity: Entity): EffectPass[] | null {+ const effects = store(world, Cache).effects[entity.id()];+ if (!effects?.length) return null;+ if (!effects.some((effect) => !effect.has(Hidden) && activeMattes(world, effect).length > 0)) return null;++ const passes: EffectPass[] = [];+ let run: string[] = [];+ const flush = () => {+ if (run.length) passes.push({ kind: 'filter', filter: run.join(' ') });+ run = [];+ };++ const blur = store(world, Computed).blur[entity.id()]!;+ if (blur > EPSILON) run.push(`blur(${blur}px)`);++ const types = store(world, Effect).type;+ for (const effect of effects) {+ if (effect.has(Hidden)) continue;+ const filter = effectFilter(world, effect);+ const mattes = activeMattes(world, effect);+ if (mattes.length === 0) {+ if (filter) run.push(filter);+ continue;+ }+ flush();+ passes.push({ kind: 'masked', filter, mattes, opacity: types[effect.id()] === EffectType.OPACITY });+ }+ flush();+ return passes;+}++const NO_MATTES: Entity[] = [];++/** The effect's mattes that have a picture and are switched on. */+function activeMattes(world: World, effect: Entity): Entity[] {+ const all = store(world, Cache).mattes[effect.id()];+ if (!all?.length) return NO_MATTES;++ let active: Entity[] | null = null;+ for (const matte of all) {+ if (matte.has(Hidden) || !matte.has(AssetId)) continue;+ (active ??= []).push(matte);+ }+ return active ?? NO_MATTES;+}++// ── Layers ───────────────────────────────────────────────────++type Layer = { canvas: OffscreenCanvas; ctx: OffscreenCanvasRenderingContext2D };++// Layers for nodes drawn through masked effects, the size of the surface: a+// pool by depth, since a masked node's children may be too.+const layers: Layer[] = [];+let layerDepth = 0;++function acquireLayer(width: number, height: number): Layer {+ let layer = layers[layerDepth];+ if (!layer) {+ const canvas = new OffscreenCanvas(width, height);+ layer = { canvas, ctx: canvas.getContext('2d')! };+ layers[layerDepth] = layer;+ } else if (layer.canvas.width !== width || layer.canvas.height !== height) {+ layer.canvas.width = width;+ layer.canvas.height = height;+ }+ layerDepth++;+ return layer;+}++/** Clears a layer and puts its state back to the defaults, under `transform`. */+function resetLayer(layer: Layer, transform: DOMMatrix): void {+ const { ctx, canvas } = layer;+ ctx.setTransform(1, 0, 0, 1, 0, 0);+ ctx.clearRect(0, 0, canvas.width, canvas.height);+ ctx.globalAlpha = 1;+ ctx.globalCompositeOperation = 'source-over';+ ctx.filter = 'none';+ ctx.setTransform(transform);+}++/** Draws `source` onto `target` pixel for pixel, composited with `operation`, through `filter`, at `alpha`. */+function blit(target: Layer, source: OffscreenCanvas, operation: GlobalCompositeOperation = 'source-over', filter = 'none', alpha = 1): void {+ const { ctx } = target;+ ctx.save();+ ctx.setTransform(1, 0, 0, 1, 0, 0);+ ctx.globalCompositeOperation = operation;+ ctx.filter = filter;+ ctx.globalAlpha = alpha;+ ctx.drawImage(source, 0, 0);+ ctx.restore();+}+const IDENTITY = new DOMMatrix();++/**+ * Draws the node through layers: its content unfiltered onto one, then+ * each effect pass from one layer to the next (a filter run is a filtered+ * copy; a masked effect combines the copy with the unfiltered picture by+ * its mattes' coverage), and the layer lands on the surface under the+ * node's opacity and blend mode, which the caller has already set.+ */+function renderLayered(world: World, entity: Entity, passes: EffectPass[]): void {+ const ctx = getCtx(world);+ const surface = world.get(RenderSurface)!.canvas!;+ const base = layerDepth;+ const local = ctx.getTransform();++ try {+ let front = acquireLayer(surface.width, surface.height);+ resetLayer(front, local);+ drawOnto(front.ctx, () => renderContent(world, entity, null));++ let back = acquireLayer(surface.width, surface.height);+ for (const pass of passes) {+ if (pass.kind === 'filter') {+ resetLayer(back, IDENTITY);+ blit(back, front.canvas, 'source-over', pass.filter);+ } else {+ applyMaskedEffect(world, entity, pass, front, back, local);+ }+ [front, back] = [back, front];+ }++ ctx.save();+ ctx.setTransform(1, 0, 0, 1, 0, 0);+ ctx.drawImage(front.canvas, 0, 0);+ ctx.restore();+ } finally {+ layerDepth = base;+ }+}++/**+ * One effect limited by its mattes: `front` holds the node so far, and the+ * outcome lands in `back`. The mattes' coverage K is the node's box less+ * what each matte takes away, weighted by the matte's opacity. A filter+ * shows through where K covers — front·(1−K) + filtered·K. The opacity+ * effect keeps only what K covers — filtered·K — so what is outside its+ * mask goes transparent, which is what a mask on Opacity means in Premiere.+ */+function applyMaskedEffect(world: World, entity: Entity, pass: Extract<EffectPass, { kind: 'masked' }>, front: Layer, back: Layer, local: DOMMatrix): void {+ const { width, height } = front.canvas;+ const coverage = acquireLayer(width, height);+ const scratch = acquireLayer(width, height);+ const opacities = store(world, Computed).opacity;++ try {+ resetLayer(coverage, local);+ drawOnto(coverage.ctx, () => drawRectPath(world, entity));+ coverage.ctx.fillStyle = '#000000';+ coverage.ctx.fill();+ for (const matte of pass.mattes) {+ const strength = Math.min(1, Math.max(0, opacities[matte.id()] ?? 1));+ if (strength <= EPSILON || !drawMatteRemoval(world, entity, matte, scratch, local)) continue;+ blit(coverage, scratch.canvas, 'destination-out', 'none', strength);+ }++ resetLayer(back, IDENTITY);+ blit(back, front.canvas, 'source-over', pass.filter ?? 'none');+ blit(back, coverage.canvas, 'destination-in');++ if (!pass.opacity) {+ blit(front, coverage.canvas, 'destination-out');+ blit(back, front.canvas, 'lighter');+ }+ } finally {+ layerDepth -= 2;+ }+}++/**+ * Draws onto `layer` what `matte` takes away from the node: the node's box+ * less the matte's picture, or the picture itself when the matte is+ * inverted, fitted into the box the way the node fits its footage so it lands+ * on the frame it was made from, and softened by the matte's blur. A mask+ * file's picture is its outline, filled at the size it lands at so its edge+ * stays sharp at any scale, smoothed as much as the matte asks. False, and nothing drawn, while the picture's frame is not decoded yet.+ */+function drawMatteRemoval(world: World, entity: Entity, matte: Entity, layer: Layer, local: DOMMatrix): boolean {+ const computed = store(world, Computed);+ const eid = entity.id();+ const mid = matte.id();+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;++ const decoder = resolveVideoDecoder(world, matte);+ const smoothing = store(world, Matte).smoothing[mid] ?? DEFAULT_MASK_SMOOTHING;+ const outline = decoder instanceof MaskDecoder ? decoder.getOutline(smoothing) : null;+ const frame = outline ? null : decoder?.toBitmap();+ const picture = outline ?? frame;+ if (!picture) return false;++ const feather = computed.blur[mid] ?? 0;+ const inverted = store(world, Matte).inverted[mid] ?? false;+ const source = findGeometryAssetSource(world, entity);+ const mode = (source && store(world, ScaleMode).value[source.id()]) ?? ScaleModeType.COVER;+ const [dx, dy, sw, sh] = getScaledImageProps(mode, picture.width, picture.height, w, h);++ const { ctx } = layer;+ resetLayer(layer, local);+ ctx.save();+ drawOnto(ctx, () => drawRectPath(world, entity));+ if (!inverted) {+ ctx.fillStyle = '#000000';+ ctx.fill();+ }+ ctx.clip();+ ctx.globalCompositeOperation = inverted ? 'source-over' : 'destination-out';+ if (feather > EPSILON) ctx.filter = `blur(${feather}px)`;+ if (outline) {+ // The path is placed in the box rather than the context scaled, so the+ // feather's blur is measured as it is for any other picture.+ const placed = new Path2D();+ placed.addPath(outline.path, new DOMMatrix([sw / outline.width, 0, 0, sh / outline.height, dx, dy]));+ ctx.fillStyle = '#000000';+ ctx.fill(placed);+ } else if (frame) {+ ctx.imageSmoothingEnabled = true;+ ctx.drawImage(frame, dx, dy, sw, sh);+ }ctx.restore();+ return true;}/**diff --git a/packages/runtime/src/traits/style.ts b/packages/runtime/src/traits/style.tsindex 171ac0de..f9975f77 100644--- a/packages/runtime/src/traits/style.ts+++ b/packages/runtime/src/traits/style.ts@@ -2,6 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */+import { DEFAULT_MASK_SMOOTHING } from '@diffusionstudio/assets';import { trait } from 'koota';import { BlendModeType, EffectType, ScaleModeType, StrokeJoin, StrokeCap } from '../constants';@@ -41,6 +42,8 @@ export const Effect = trait({export const Matte = trait({offset: 0,inverted: false,+ // How much a mask file's edge is smoothed, 0 to 1 (see `traceMask`).+ smoothing: DEFAULT_MASK_SMOOTHING,});// Single gradient stop: its position along the gradient (0-1). Its color anddiff --git a/packages/runtime/src/utils/surface.ts b/packages/runtime/src/utils/surface.tsnew file mode 100644index 00000000..252bad63--- /dev/null+++ b/packages/runtime/src/utils/surface.ts@@ -0,0 +1,31 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { RenderSurface } from '../traits';++import type { World } from 'koota';++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;++// The layer a node is being drawn onto instead of the surface while a soft+// mask is cut from it (see the render system's `renderThroughMasks`). Module+// state rather than a trait write: a world write is a change event, and the+// swap happens twice per masked node per frame.+let redirected: Ctx2D | null = null;++/** Where drawing goes: the surface's context, or the layer a masked node is being drawn onto. */+export function getSurfaceContext(world: World): Ctx2D | null {+ return redirected ?? world.get(RenderSurface)?.ctx ?? null;+}++/** Runs `draw` with everything that draws through `getSurfaceContext` sent to `ctx`. Nests. */+export function drawOnto<T>(ctx: Ctx2D, draw: () => T): T {+ const previous = redirected;+ redirected = ctx;+ try {+ return draw();+ } finally {+ redirected = previous;+ }+}diff --git a/packages/runtime/src/utils/text.ts b/packages/runtime/src/utils/text.tsindex 9ba4a201..ba28d59a 100644--- a/packages/runtime/src/utils/text.ts+++ b/packages/runtime/src/utils/text.ts@@ -17,6 +17,7 @@ import {import { clamp } from '../math/common';import { colorToHex } from './color';import { applyStrokeStyle, findWidestStroke } from './stroke';+import { getSurfaceContext } from './surface';import { createLinearGradient, createRadialGradient } from '../systems/gradients';import type { Entity, World } from 'koota';@@ -434,7 +435,7 @@ function renderTokens(ctx: Ctx, world: World, entity: Entity): void {/** Render text tokens directly to the world's render surface. */export function renderText(world: World, entity: Entity) {- const ctx = world.get(RenderSurface)?.ctx;+ const ctx = getSurfaceContext(world);if (!ctx) return;tokenizeText(world, entity);shapeTokens(world, entity);diff --git a/packages/runtime/src/world/serialize.ts b/packages/runtime/src/world/serialize.tsindex f456a130..032c7b11 100644--- a/packages/runtime/src/world/serialize.ts+++ b/packages/runtime/src/world/serialize.ts@@ -44,6 +44,7 @@ export interface EntityRecord {Matte?: {offset?: number;inverted?: boolean;+ smoothing?: number;};Caption?: {type?: number;@@ -205,7 +206,7 @@ export function serializeEntity(entity: Entity): EntityRecord {}if (entity.has(Matte)) {const matte = entity.get(Matte)!;- record.Matte = { offset: matte.offset, inverted: matte.inverted };+ record.Matte = { offset: matte.offset, inverted: matte.inverted, smoothing: matte.smoothing };}if (entity.has(Shadow)) {record.Shadow = {};diff --git a/packages/runtime/tsconfig.json b/packages/runtime/tsconfig.jsonindex 516eda7b..50672c68 100644--- a/packages/runtime/tsconfig.json+++ b/packages/runtime/tsconfig.json@@ -4,7 +4,7 @@"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo","erasableSyntaxOnly": false,"lib": ["es2024", "DOM", "DOM.Iterable"],- "types": ["@webgpu/types"]+ "types": ["@webgpu/types", "@types/wicg-file-system-access"]},"include": ["src"]}diff --git a/packages/sam2/src/video.ts b/packages/sam2/src/video.tsindex fa358379..47c42e6d 100644--- a/packages/sam2/src/video.ts+++ b/packages/sam2/src/video.ts@@ -6,7 +6,7 @@ import { VideoSampleSink } from 'mediabunny';import type { InputVideoTrack, VideoSample } from 'mediabunny';import type { Sam2Mask } from './mask';-import type { Sam2Point, Sam2Video } from './tracker';+import type { Sam2Correction, Sam2Point, Sam2Video } from './tracker';export type FrameRequest = {track: InputVideoTrack;@@ -20,6 +20,8 @@ export type TrackRequest = {/** Which of `timestamps` the points were placed on. */seedIndex: number;points: Sam2Point[];+ /** The seed's mask as the user corrected it: what the object is tracked from. */+ correct?: Sam2Correction;signal?: AbortSignal;/** Called as each frame's mask is ready, in tracking order: the seed, then forward, then backward. */onMask: (index: number, mask: Sam2Mask) => void;@@ -50,7 +52,7 @@ export async function holdFrame(model: Sam2Video, request: FrameRequest): Promis* its mask instead of being run again.*/export async function trackObject(model: Sam2Video, request: TrackRequest): Promise<void> {- const { track, timestamps, seedIndex, points, signal, onMask } = request;+ const { track, timestamps, seedIndex, points, correct, signal, onMask } = request;const sink = new VideoSampleSink(track);const rotation = track.rotation;const total = timestamps.length;@@ -59,7 +61,7 @@ export async function trackObject(model: Sam2Video, request: TrackRequest): Promif (!seed) throw new Error('The prompted frame could not be decoded');const seedTimestamp = seed.timestamp;- const seedMask = await withVideoFrame(seed, (frame) => model.seed(frame, rotation, points, seedIndex));+ const seedMask = await withVideoFrame(seed, (frame) => model.seed(frame, rotation, points, seedIndex, correct));onMask(seedIndex, seedMask);let previous = { timestamp: seedTimestamp, mask: seedMask };