The VS Code of video editing, built for agents.
TypeScript
Implement Object Mask Tool and Enhance UI Components
konstantin-pauluse63f1c7a0f1b
diff --git a/apps/web/package.json b/apps/web/package.jsonindex 5835aefd..00f7d107 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -22,6 +22,7 @@"@diffusionstudio/koota-solid": "*","@diffusionstudio/reconciler": "*","@diffusionstudio/runtime": "*",+ "@diffusionstudio/sam2": "*","@kobalte/core": "^0.13.11","@sentry/solid": "^10.51.0","@solid-primitives/keyboard": "^1.3.3",diff --git a/apps/web/src/assets/icons/tool.object-mask.svg b/apps/web/src/assets/icons/tool.object-mask.svgnew file mode 100644index 00000000..00c33422--- /dev/null+++ b/apps/web/src/assets/icons/tool.object-mask.svg@@ -0,0 +1,8 @@+<svg viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">+<path d="M12 4.5C9.51472 4.5 7.5 6.51472 7.5 9C7.5 11.4853 9.51472 13.5 12 13.5C14.4853 13.5 16.5 11.4853 16.5 9C16.5 6.51472 14.4853 4.5 12 4.5ZM8.625 9C8.625 7.13604 10.136 5.625 12 5.625C13.864 5.625 15.375 7.13604 15.375 9C15.375 10.864 13.864 12.375 12 12.375C10.136 12.375 8.625 10.864 8.625 9Z" fill="currentColor"/>+<path d="M12 14.625C8.89 14.625 6.5 16.4 5.4 19.5H18.6C17.5 16.4 15.11 14.625 12 14.625ZM12 15.75C14.2 15.75 15.9 16.85 16.9 18.375H7.1C8.1 16.85 9.8 15.75 12 15.75Z" fill="currentColor"/>+<path d="M4.5 6.5V5.5C4.5 4.94772 4.94772 4.5 5.5 4.5H6.5" stroke="currentColor" stroke-width="1.125" stroke-linecap="round" stroke-dasharray="1.5 1.5"/>+<path d="M17.5 4.5H18.5C19.0523 4.5 19.5 4.94772 19.5 5.5V6.5" stroke="currentColor" stroke-width="1.125" stroke-linecap="round" stroke-dasharray="1.5 1.5"/>+<path d="M19.5 17.5V18.5C19.5 19.0523 19.0523 19.5 18.5 19.5H17.5" stroke="currentColor" stroke-width="1.125" stroke-linecap="round" stroke-dasharray="1.5 1.5"/>+<path d="M6.5 19.5H5.5C4.94772 19.5 4.5 19.0523 4.5 18.5V17.5" stroke="currentColor" stroke-width="1.125" stroke-linecap="round" stroke-dasharray="1.5 1.5"/>+</svg>diff --git a/apps/web/src/components/canvas/toolbar.tsx b/apps/web/src/components/canvas/toolbar.tsxindex bd4be2aa..bdbb1cc8 100644--- a/apps/web/src/components/canvas/toolbar.tsx+++ b/apps/web/src/components/canvas/toolbar.tsx@@ -16,16 +16,23 @@ import {} from "@/components/ui/dropdown-menu";import { PromptInput } from "../genai/prompt-input";import { ActionBar } from "../genai/action-bar";-import { Show } from "solid-js";+import { For, Show, createMemo } from "solid-js";import { Tool, ToolType } from "@diffusionstudio/runtime";import { useWorld } from "@diffusionstudio/koota-solid";import { useTool } from "@/engine";import { usePromptInput } from "@/context/prompt-input";+const CURSOR_TOOLS = [+ { tool: ToolType.MOVE, label: 'Move', shortcut: 'V', icon: 'move', menuIcon: 'move-small' },+ { tool: ToolType.HAND, label: 'Hand', shortcut: 'H', icon: 'hand', menuIcon: 'hand' },+ { tool: ToolType.OBJECT_MASK, label: 'Object Mask', shortcut: 'M', icon: 'tool.object-mask', menuIcon: 'tool.object-mask' },+] as const;+export function Toolbar() {const world = useWorld();const { promptInputOpen, promptInputConfig, openPromptInput, setPromptInputOpen } = usePromptInput();const selectedTool = useTool();+ const cursorTool = createMemo(() => CURSOR_TOOLS.find((cursor) => cursor.tool === selectedTool()));const handleToolChange = (tool: ToolType) => {world.set(Tool, { value: tool });@@ -45,16 +52,14 @@ export function Toolbar() {<TooltipTriggeras={Button}size="icon-square"- class={[ToolType.MOVE, ToolType.HAND].includes(selectedTool()) ? 'text-foreground' : 'text-muted-foreground'}- variant={[ToolType.MOVE, ToolType.HAND].includes(selectedTool()) ? 'default' : 'ghost'}- onClick={() => handleToolChange(- selectedTool() === ToolType.HAND ? ToolType.HAND : ToolType.MOVE- )}+ class={cursorTool() ? 'text-foreground' : 'text-muted-foreground'}+ variant={cursorTool() ? 'default' : 'ghost'}+ onClick={() => handleToolChange(cursorTool()?.tool ?? ToolType.MOVE)}>- <Icon name={selectedTool() === ToolType.HAND ? 'hand' : 'move'} />+ <Icon name={(cursorTool() ?? CURSOR_TOOLS[0]!).icon} /></TooltipTrigger>- <TooltipContent shortcut={selectedTool() === ToolType.HAND ? 'H' : 'V'}>- {selectedTool() === ToolType.HAND ? 'Hand' : 'Move'}+ <TooltipContent shortcut={(cursorTool() ?? CURSOR_TOOLS[0]!).shortcut}>+ {(cursorTool() ?? CURSOR_TOOLS[0]!).label}</TooltipContent></Tooltip><DropdownMenu placement="top-start">@@ -75,22 +80,18 @@ export function Toolbar() {</Tooltip><DropdownMenuPortal><DropdownMenuContent>- <DropdownMenuItem class="px-0 pr-2 gap-0.5" onSelect={() => handleToolChange(ToolType.MOVE)}>- <div classList={{ "visible": selectedTool() === ToolType.MOVE }} class="invisible">- <Icon name="confirm-check" class="text-foreground" />- </div>- <Icon name="move-small" class="text-foreground" />- <span class="min-w-12 mx-1">Move</span>- <DropdownMenuShortcut>V</DropdownMenuShortcut>- </DropdownMenuItem>- <DropdownMenuItem class="px-0 pr-2 gap-0.5" onSelect={() => handleToolChange(ToolType.HAND)}>- <div classList={{ "visible": selectedTool() === ToolType.HAND }} class="invisible">- <Icon name="confirm-check" class="text-foreground" />- </div>- <Icon name="hand" class="text-foreground" />- <span class="min-w-12 mx-1">Hand</span>- <DropdownMenuShortcut>H</DropdownMenuShortcut>- </DropdownMenuItem>+ <For each={CURSOR_TOOLS}>+ {(cursor) => (+ <DropdownMenuItem class="px-0 pr-2 gap-0.5" onSelect={() => handleToolChange(cursor.tool)}>+ <div classList={{ "visible": selectedTool() === cursor.tool }} class="invisible">+ <Icon name="confirm-check" class="text-foreground" />+ </div>+ <Icon name={cursor.menuIcon} class="text-foreground" />+ <span class="min-w-12 mx-1">{cursor.label}</span>+ <DropdownMenuShortcut>{cursor.shortcut}</DropdownMenuShortcut>+ </DropdownMenuItem>+ )}+ </For></DropdownMenuContent></DropdownMenuPortal></DropdownMenu>diff --git a/apps/web/src/components/sidebar-left/project-menu/tool-menu.tsx b/apps/web/src/components/sidebar-left/project-menu/tool-menu.tsxindex d17449f5..d3e378c2 100644--- a/apps/web/src/components/sidebar-left/project-menu/tool-menu.tsx+++ b/apps/web/src/components/sidebar-left/project-menu/tool-menu.tsx@@ -40,6 +40,10 @@ export function ToolMenu() {Rectangle<DropdownMenuShortcut>R</DropdownMenuShortcut></DropdownMenuItem>+ <DropdownMenuItem onSelect={() => setTool(ToolType.OBJECT_MASK)}>+ Object Mask+ <DropdownMenuShortcut>M</DropdownMenuShortcut>+ </DropdownMenuItem></DropdownMenuGroup></>);diff --git a/apps/web/src/components/sidebar-right/inspector/effect-types.ts b/apps/web/src/components/sidebar-right/inspector/effect-types.tsindex 67d96204..e1593809 100644--- a/apps/web/src/components/sidebar-right/inspector/effect-types.ts+++ b/apps/web/src/components/sidebar-right/inspector/effect-types.ts@@ -33,6 +33,7 @@ export const EFFECT_OPTIONS: EffectOption[] = [{ name: "invert", label: "Invert", unit: "amount", value: 0.5 },{ name: "saturate", label: "Saturate", unit: "amount", value: 0.8 },{ name: "sepia", label: "Sepia", unit: "amount", value: 0.5 },+ { name: "opacity", label: "Opacity", unit: "amount", value: 1 },];/** What the panel's plus inserts, spelled out. */diff --git a/apps/web/src/components/sidebar-right/inspector/inspector.tsx b/apps/web/src/components/sidebar-right/inspector/inspector.tsxindex eb05b23a..ade8a7c5 100644--- a/apps/web/src/components/sidebar-right/inspector/inspector.tsx+++ b/apps/web/src/components/sidebar-right/inspector/inspector.tsx@@ -22,6 +22,7 @@ import { InspectorHeader } from "./inspector-header";import { BackgroundSettings } from "./background";import { VariablesSettings } from "./variables";import { SceneTemplatePanel } from "./scene-template";+import { ObjectMaskPanel } from "./object-mask";import { AssetInfoPanel } from "./asset-info";import { TimeSettings } from "./time";import { AppearanceSettings } from "./appearance";@@ -46,6 +47,7 @@ import type { Entity } from "koota";export type SelectionTarget =| "scene-tool"+ | "mask-tool"| "keyframe"| "asset"| "scene"@@ -87,6 +89,7 @@ export function Inspector() {const selectionTarget = createMemo<SelectionTarget>(() => {if (tool() === ToolType.SCENE) return "scene-tool";+ if (tool() === ToolType.OBJECT_MASK) return "mask-tool";if (keyframes().length > 0) return "keyframe";if (asset() || partial()) return "asset";const entity = first();@@ -112,6 +115,10 @@ export function Inspector() {<SceneTemplatePanel /></Show>+ <Show when={includesTarget("mask-tool")}>+ <ObjectMaskPanel />+ </Show>+<Show when={nodes().length > 1}><Alignment /></Show>diff --git a/apps/web/src/engine/create-engine.ts b/apps/web/src/engine/create-engine.tsindex a5f23bca..4ca9c2aa 100644--- a/apps/web/src/engine/create-engine.ts+++ b/apps/web/src/engine/create-engine.ts@@ -8,6 +8,7 @@ import { createSignal, type Accessor, type Setter } from 'solid-js';import { AssetSelection, Hud, Keys, MODIFIER_KEYS, Pointer, PointerEvents, ProjectConfig, SnapLines } from './traits';import { inputSystem } from './input/input-system';import { clearClipFrames, clearClipPeaks, clearMedia, clearPeaks, timelineSystem, TimelineSurface } from './timeline';+import { clearObjectTrackOf, clearObjectTracks } from './object-mask';import { shortcutSystem } from './input/shortcuts';import type { RuntimeWorld } from '@diffusionstudio/runtime';@@ -59,6 +60,7 @@ class Engine {this.world.onRemove(Geometry, (entity) => {clearClipPeaks(entity.id());clearClipFrames(entity.id());+ clearObjectTrackOf(entity);}),);@@ -283,6 +285,7 @@ class Engine {// files, and are held outside the world: they go with the engine.clearPeaks();clearMedia();+ clearObjectTracks();this.unsubscribeEventListeners();this.unsubscribe.forEach(unsubscribe => unsubscribe());diff --git a/apps/web/src/engine/index.ts b/apps/web/src/engine/index.tsindex c00e81d1..1aed3a88 100644--- a/apps/web/src/engine/index.ts+++ b/apps/web/src/engine/index.ts@@ -23,3 +23,4 @@ export * from './library';export * from './asset-actions';export * from './keyframes';export * from './align';+export * from './object-mask';diff --git a/apps/web/src/engine/input/cursor.ts b/apps/web/src/engine/input/cursor.tsindex c7aae79e..ac8cf5b4 100644--- a/apps/web/src/engine/input/cursor.ts+++ b/apps/web/src/engine/input/cursor.ts@@ -89,6 +89,7 @@ const TOOL_CURSORS: Record<ToolType, { idle: CursorType; pressed?: CursorType }>[ToolType.RECT]: { idle: 'cross' },[ToolType.TEXT]: { idle: 'cross' },[ToolType.TEXT_EDIT]: { idle: 'text' },+ [ToolType.OBJECT_MASK]: { idle: 'cross' },};/** The cursor the armed tool asks for, pressed or at rest. */diff --git a/apps/web/src/engine/input/shortcuts.ts b/apps/web/src/engine/input/shortcuts.tsindex 59c7e083..21ac53b8 100644--- a/apps/web/src/engine/input/shortcuts.ts+++ b/apps/web/src/engine/input/shortcuts.ts@@ -470,6 +470,7 @@ const PRESSED_SHORTCUTS: readonly Shortcut[] = [{ keys: ['f', '!mod'], action: selectTool(ToolType.SCENE) },{ keys: ['t', '!mod'], action: selectTool(ToolType.TEXT) },{ keys: ['r', '!mod'], action: selectTool(ToolType.RECT) },+ { keys: ['m', '!mod'], action: selectTool(ToolType.OBJECT_MASK) },{ keys: ['a', '!mod'], action: seekFrames(-1) },{ keys: ['d', '!mod'], action: seekFrames(1) },{ keys: ['w', '!mod'], action: seekSeconds(1) },diff --git a/docs/reference/jsx/elements.md b/docs/reference/jsx/elements.mdindex 07081902..92f7a2af 100644--- a/docs/reference/jsx/elements.md+++ b/docs/reference/jsx/elements.md@@ -49,6 +49,7 @@ Sub-entity children of the node (or paint) that holds them. See [styles.md](./st| `<stroke>` | An outline of the parent's box or glyphs. Several stack. || `<shadow>` | A drop shadow beneath the parent's silhouette. Several stack. || `<effect>` | A CSS-style filter over the parent's rendered pixels. Several stack. |+| `<mask>` | A matte limiting the `<effect>` holding it: the alpha of a tracked frame sequence. Under an `opacity` effect, the cut-out. See [styles.md](./styles.md#mask). || `<animation>` | One preset in/out animation of the node holding it. || `<keyframeTrack>` | The keyframes of one prop of the element holding it. || `<keyframe>` | One keyframe of the track holding it. |diff --git a/docs/reference/jsx/styles.md b/docs/reference/jsx/styles.mdindex 4e10e161..4f69757b 100644--- a/docs/reference/jsx/styles.md+++ b/docs/reference/jsx/styles.md@@ -39,7 +39,7 @@ A drop shadow beneath the parent's silhouette: a blurred, offset copy of it in `## `<effect>`-A filter over the parent's **rendered pixels** — its fills, strokes and children together. On a [`<group>`](./group.md) that is the group as a whole; on a [`<video>`](./video.md) it is the frame after the media is drawn.+A filter over the parent's **rendered pixels** — its fills, strokes and children together. On a [`<group>`](./group.md) that is the group as a whole; on a [`<video>`](./video.md) it is the frame after the media is drawn. A [`<mask>`](#mask) child limits where the effect applies.| Prop | Type | Default | Meaning || ---- | ---- | ------- | ------- |@@ -52,6 +52,7 @@ A filter over the parent's **rendered pixels** — its fills, strokes and childr| `"blur"` | Radius in px. || `"hueRotate"` | Degrees. || `"brightness"`, `"contrast"`, `"grayscale"`, `"invert"`, `"saturate"`, `"sepia"` | Amount, `0`–`1`. |+| `"opacity"` | The node's opacity as an effect, `0`–`1`. With a `<mask>` under it, what is outside the mask goes transparent: the cut-out. |```tsx<image src="stills/photo.jpg" width={1920} height={1080}>@@ -64,6 +65,33 @@ A filter over the parent's **rendered pixels** — its fills, strokes and childr</image>```++## `<mask>`++A matte limiting the `<effect>` holding it: a **picture** whose alpha says where the effect applies, fitted into the node's box the way the node fits its footage. It is what the editor's object mask tool makes — pick an object in a video clip, track it, and the frames it found are written to the library and named by `src`. Several under one effect intersect. Without a `src` a `<mask>` does nothing.++Under an `"opacity"` effect the mask is the cut-out: the clip shows inside the mask and goes transparent outside, as a mask on Premiere's Opacity does. Under any other effect the effect shows through inside the mask and the picture is untouched outside: an inverted mask on a `"blur"` blurs the background behind a person.++| Prop | Type | Default | Meaning |+| ---- | ---- | ------- | ------- |+| `src` | `string` | none | The frames, a directory of numbered images whose alpha is the mask. |+| `sourceIn` | `Time` | `0` | The node's source time the first frame belongs to, so the mask stays on the footage it was made from whatever the trim. |+| `frameRate` | `number` | `30` | Frames per second the frames were written at (the composition's). |+| `blur` | `number` | `0` | Feather: radius in px the edge falls off over. |+| `opacity` | `number` | `1` | How strongly the mask limits the effect. `1` stops the effect at the edge; lower lets that much of it through outside; `0` is no mask. |+| `inverted` | `boolean` | `false` | Covers what the picture does not instead. |+| `hidden` | `boolean` | absent | Switches the mask off without removing it. |++`blur` and `opacity` take a [`<keyframeTrack>`](./keyframes.md).++```tsx+<video src="footage/skater.mp4" width={1920} height={1080}>+ <effect type="opacity" value={1}>+ <mask src="masks/skater" sourceIn={2} frameRate={30} blur={6} />+ </effect>+</video>+```+## Animating themAll three take [`<keyframeTrack>`](./keyframes.md) children, and the track's `property` is read against its holder: `width` under a `<stroke>` is the line width, `blur` / `offsetX` / `offsetY` under a `<shadow>` are the shadow's, `value` under an `<effect>` is its amount, and `color` / `opacity` are whatever the holder's are.diff --git a/package-lock.json b/package-lock.jsonindex 8ef38f60..7d2ae45d 100644--- a/package-lock.json+++ b/package-lock.json@@ -100,6 +100,7 @@"@diffusionstudio/koota-solid": "*","@diffusionstudio/reconciler": "*","@diffusionstudio/runtime": "*",+ "@diffusionstudio/sam2": "*","@kobalte/core": "^0.13.11","@sentry/solid": "^10.51.0","@solid-primitives/keyboard": "^1.3.3",@@ -818,6 +819,10 @@"resolved": "packages/runtime","link": true},+ "node_modules/@diffusionstudio/sam2": {+ "resolved": "packages/sam2",+ "link": true+ },"node_modules/@diffusionstudio/web": {"resolved": "apps/web","link": true@@ -3136,6 +3141,63 @@"@octokit/openapi-types": "^12.11.0"}},+ "node_modules/@protobufjs/aspromise": {+ "version": "1.1.2",+ "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",+ "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",+ "license": "BSD-3-Clause"+ },+ "node_modules/@protobufjs/base64": {+ "version": "1.1.2",+ "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",+ "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",+ "license": "BSD-3-Clause"+ },+ "node_modules/@protobufjs/codegen": {+ "version": "2.0.5",+ "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz",+ "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==",+ "license": "BSD-3-Clause"+ },+ "node_modules/@protobufjs/eventemitter": {+ "version": "1.1.1",+ "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz",+ "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==",+ "license": "BSD-3-Clause"+ },+ "node_modules/@protobufjs/fetch": {+ "version": "1.1.1",+ "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz",+ "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==",+ "license": "BSD-3-Clause",+ "dependencies": {+ "@protobufjs/aspromise": "^1.1.1"+ }+ },+ "node_modules/@protobufjs/float": {+ "version": "1.0.2",+ "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",+ "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",+ "license": "BSD-3-Clause"+ },+ "node_modules/@protobufjs/path": {+ "version": "1.1.2",+ "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",+ "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",+ "license": "BSD-3-Clause"+ },+ "node_modules/@protobufjs/pool": {+ "version": "1.1.0",+ "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",+ "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",+ "license": "BSD-3-Clause"+ },+ "node_modules/@protobufjs/utf8": {+ "version": "1.1.2",+ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz",+ "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==",+ "license": "BSD-3-Clause"+ },"node_modules/@rollup/rollup-android-arm-eabi": {"version": "4.62.2","resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz",@@ -4484,7 +4546,6 @@"version": "24.13.2","resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.2.tgz","integrity": "sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==",- "dev": true,"license": "MIT","dependencies": {"undici-types": "~7.18.0"@@ -7958,6 +8019,12 @@"node": ">=16"}},+ "node_modules/flatbuffers": {+ "version": "25.9.23",+ "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz",+ "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==",+ "license": "Apache-2.0"+ },"node_modules/flatted": {"version": "3.4.2","resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz",@@ -8387,6 +8454,12 @@"dev": true,"license": "ISC"},+ "node_modules/guid-typescript": {+ "version": "1.0.9",+ "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",+ "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",+ "license": "ISC"+ },"node_modules/has-flag": {"version": "4.0.0","resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",@@ -9800,6 +9873,12 @@"url": "https://github.com/chalk/wrap-ansi?sponsor=1"}},+ "node_modules/long": {+ "version": "5.3.2",+ "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",+ "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",+ "license": "Apache-2.0"+ },"node_modules/lowercase-keys": {"version": "2.0.0","resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",@@ -10589,6 +10668,26 @@"url": "https://github.com/sponsors/sindresorhus"}},+ "node_modules/onnxruntime-common": {+ "version": "1.30.0",+ "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz",+ "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==",+ "license": "MIT"+ },+ "node_modules/onnxruntime-web": {+ "version": "1.30.0",+ "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.30.0.tgz",+ "integrity": "sha512-q0y+JrrtukXSzsBWEMccVfqX25LRmosXHF+CaRJmg8pZClzcV7svNc4rKY3jL02Vb7QmRMDs1SigqR4CXAfKYQ==",+ "license": "MIT",+ "dependencies": {+ "flatbuffers": "^25.1.24",+ "guid-typescript": "^1.0.9",+ "long": "^5.2.3",+ "onnxruntime-common": "1.30.0",+ "platform": "^1.3.6",+ "protobufjs": "^7.2.4"+ }+ },"node_modules/open": {"version": "7.4.2","resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz",@@ -11040,6 +11139,12 @@"node": ">=16.20.0"}},+ "node_modules/platform": {+ "version": "1.3.6",+ "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",+ "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",+ "license": "MIT"+ },"node_modules/plist": {"version": "3.1.1","resolved": "https://registry.npmjs.org/plist/-/plist-3.1.1.tgz",@@ -11177,6 +11282,29 @@"node": ">=10"}},+ "node_modules/protobufjs": {+ "version": "7.6.6",+ "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz",+ "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==",+ "hasInstallScript": true,+ "license": "BSD-3-Clause",+ "dependencies": {+ "@protobufjs/aspromise": "^1.1.2",+ "@protobufjs/base64": "^1.1.2",+ "@protobufjs/codegen": "^2.0.5",+ "@protobufjs/eventemitter": "^1.1.1",+ "@protobufjs/fetch": "^1.1.1",+ "@protobufjs/float": "^1.0.2",+ "@protobufjs/path": "^1.1.2",+ "@protobufjs/pool": "^1.1.0",+ "@protobufjs/utf8": "^1.1.1",+ "@types/node": ">=13.7.0",+ "long": "^5.3.2"+ },+ "engines": {+ "node": ">=12.0.0"+ }+ },"node_modules/proxy-addr": {"version": "2.0.7","resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",@@ -13068,7 +13196,6 @@"version": "7.18.2","resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz","integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",- "dev": true,"license": "MIT"},"node_modules/unique-filename": {@@ -14142,6 +14269,19 @@"@webgpu/types": "^0.1.71","typescript": "~5.9.3"}+ },+ "packages/sam2": {+ "name": "@diffusionstudio/sam2",+ "version": "0.1.0",+ "license": "MPL-2.0",+ "dependencies": {+ "mediabunny": "^1.32.2",+ "onnxruntime-web": "^1.30.0"+ },+ "devDependencies": {+ "@webgpu/types": "^0.1.71",+ "typescript": "~5.9.3"+ }}}}diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.tsindex 8c74aab1..4aebee5d 100644--- a/packages/jsx/src/index.ts+++ b/packages/jsx/src/index.ts@@ -69,6 +69,7 @@ export type {ColorStopProps,Easing,EffectProps,+ MaskProps,EffectType,Fit,GradientPaintProps,diff --git a/packages/jsx/src/jsx-runtime.ts b/packages/jsx/src/jsx-runtime.tsindex e2478b6e..413ec625 100644--- a/packages/jsx/src/jsx-runtime.ts+++ b/packages/jsx/src/jsx-runtime.ts@@ -27,6 +27,7 @@ import type {CaptionsProps,ColorStopProps,EffectProps,+ MaskProps,GradientPaintProps,MediaPaintProps,GroupProps,@@ -98,6 +99,7 @@ export declare namespace JSX {stroke: StrokeProps & SourceProps;shadow: ShadowProps & SourceProps;effect: EffectProps & SourceProps;+ mask: MaskProps & SourceProps;animation: AnimationProps & SourceProps;keyframeTrack: KeyframeTrackProps & SourceProps;keyframe: KeyframeProps & SourceProps;diff --git a/packages/jsx/src/source.ts b/packages/jsx/src/source.tsindex 2e6fd10a..5610f831 100644--- a/packages/jsx/src/source.ts+++ b/packages/jsx/src/source.ts@@ -110,6 +110,7 @@ export const COMPOSITION_TAGS = ["stroke","shadow","effect",+ "mask","animation","keyframeTrack","keyframe",diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 28995c62..16381a62 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -59,7 +59,9 @@ export type BlendMode =/*** An `<effect>`'s filter — the CSS filter functions, applied to the parent's* rendered pixels. `blur` takes a radius in px, `hueRotate` degrees, the- * rest an amount 0–1.+ * rest an amount 0–1. `opacity` is the node's opacity as an effect (0–1,+ * default 1): on its own it is the node's `opacity` prop; with a `<mask>`+ * under it, what is outside the mask goes transparent — the cut-out.*/export type EffectType =| "blur"@@ -69,7 +71,8 @@ export type EffectType =| "hueRotate"| "invert"| "saturate"- | "sepia";+ | "sepia"+ | "opacity";/*** Easing for the segment from a keyframe to the next one: a named preset or@@ -590,13 +593,43 @@ export type ShadowProps = ColorProps & OpacityProps & Pick<CompositeProps, "hidd/*** `<effect>` — a filter over the parent's rendered pixels (its fills, strokes* and children together), a sub-entity like a paint. Several stack in- * document order.+ * document order. `<mask>` children limit where the effect applies.*/-export type EffectProps = Pick<CompositeProps, "hidden"> & TrackChildren & {+export type EffectProps = Pick<CompositeProps, "hidden"> & {/** Which filter to apply. */type: EffectType;/** The amount: px for "blur", degrees for "hueRotate", 0–1 otherwise. */value: number;+ /** `<Mask>` children limiting the effect, and `<KeyframeTrack>` children. */+ children?: SolidJSX.Element;+};++/**+ * `<mask>` — a matte limiting the `<effect>` holding it: a picture whose+ * alpha says where the effect applies, fitted into the node's box the way+ * the node fits its footage. Under an `"opacity"` effect it is the cut-out:+ * what is outside the mask goes transparent. What the editor's object mask+ * tool makes: it tracks an object through a video and writes the frames it+ * found as a directory of numbered images, which `src` names. Several under+ * one effect intersect. Without a `src` a mask does nothing.+ */+export type MaskProps = Pick<CompositeProps, "hidden"> & TrackChildren & {+ /** The matte's frames: a directory of numbered images whose alpha is the mask. */+ src?: string;+ /** The parent's source time the first frame belongs to, so the frames stay on the footage they were made from whatever the trim. Default 0. */+ sourceIn?: Time;+ /** Frames per second the frames were written at (the composition's). Default 30. */+ frameRate?: number;+ /** Feather: radius in px the edge falls off over. Default 0. */+ blur?: number;+ /**+ * How strongly the mask limits the effect, 0–1. At 1 the effect stops at+ * the mask's edge; lower lets that much of it through outside; 0 is no+ * mask. Default 1.+ */+ opacity?: number;+ /** Covers what the picture does not instead. Default false. */+ inverted?: boolean;};/**diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex ad35ec7b..4717d76d 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, Constraint, ConstraintCache, ConstraintType, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, setCameraMatrix, setPlayhead, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, Constraint, ConstraintCache, ConstraintType, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Matte, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, setCameraMatrix, setPlayhead, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import { createSignal } from 'solid-js';import { SVGElements } from 'solid-js/web';@@ -290,6 +290,7 @@ export const EFFECT_TYPES: Record<string, EffectType> = {invert: EffectType.INVERT,saturate: EffectType.SATURATE,sepia: EffectType.SEPIA,+ opacity: EffectType.OPACITY,};const STROKE_JOINS: Record<string, StrokeJoin> = {@@ -666,6 +667,11 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {entity.set(Effect, { type: EffectType.LAYER_BLUR, value: 0 });break;}+ case 'mask': {+ entity = createEntity(this.world);+ entity.add(Matte);+ break;+ }case 'animation': {entity = createEntity(this.world);entity.add(Animation);@@ -963,6 +969,11 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {}return;}+ case 'inverted': {+ if (!entity.has(Matte)) return;+ entity.set(Matte, { inverted: value === true });+ return;+ }case 'keepAspectRatio': {this.syncAspectLock(node);return;@@ -1131,6 +1142,13 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {case 'end':case 'sourceIn':case 'sourceOut': {+ if (entity.has(Matte)) {+ // A matte's frames start at this source time of its parent.+ if (name === 'sourceIn') {+ entity.set(Matte, { offset: this.toFrames(toSeconds(value) ?? 0) })+ };+ return;+ }if (entity.has(TextRange)) {// A range's start/end are character indices, not times; an// unset end runs to the end of the text.@@ -1168,7 +1186,8 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {// superseded by whatever this one starts.entity.remove(GenerationRequest, LoadRequest, PendingSource, Generating);- if (value === undefined || value === null || value === '') {+ // `false` is how an editor unsets a prop (the writer drops the attribute).+ if (value === undefined || value === null || value === false || value === '') {entity.remove(AssetId);// A `<captions>` without a src transcribes its scene instead.if (entity.has(Caption)) {diff --git a/packages/reconciler/src/elements.ts b/packages/reconciler/src/elements.tsindex 76d0af9b..c0bf150a 100644--- a/packages/reconciler/src/elements.ts+++ b/packages/reconciler/src/elements.ts@@ -25,6 +25,7 @@ import type {CaptionsProps,ColorStopProps,EffectProps,+ MaskProps,GradientPaintProps,GroupProps,HtmlPaintProps,@@ -106,6 +107,7 @@ export const ColorStop = hostElement<ColorStopProps>("ColorStop");export const Stroke = hostElement<StrokeProps>("Stroke");export const Shadow = hostElement<ShadowProps>("Shadow");export const Effect = hostElement<EffectProps>("Effect");+export const Mask = hostElement<MaskProps>("Mask");export const Animation = hostElement<AnimationProps>("Animation");export const KeyframeTrack = hostElement<KeyframeTrackProps>("KeyframeTrack");export const Keyframe = hostElement<KeyframeProps>("Keyframe");diff --git a/packages/runtime/src/actions/cache.ts b/packages/runtime/src/actions/cache.tsindex eb509237..9e2d4e5f 100644--- a/packages/runtime/src/actions/cache.ts+++ b/packages/runtime/src/actions/cache.ts@@ -11,7 +11,7 @@ import { Not, Or } from 'koota';import { store } from '../world/store';import {ChildOf, Cache,- Geometry, Group, AdjustmentLayer, IsMask, Paint, Stroke, Shadow, Effect,+ Geometry, Group, AdjustmentLayer, IsMask, Matte, Paint, Stroke, Shadow, Effect,TextRange, KeyframeTrack, Keyframe, Animation,} from '../traits';import { isStage } from '../queries/predicates';@@ -48,6 +48,11 @@ export function rebuildCaches(world: World, entity: Entity, parent: Entity | nul.sort(sortByItemIndex);}+ if (entity.has(Matte)) {+ cache.mattes[pid] = collect(Matte, ChildOf(parent))+ .sort(sortByItemIndex);+ }+// A track attached directly, or carried by a sub-entity (a paint, stroke,// effect...) that is attached with tracks already under it: JSX builds// bottom-up, so a paint's tracks exist before the paint meets its node.@@ -100,7 +105,7 @@ export function rebuildCaches(world: World, entity: Entity, parent: Entity | nul/** Every list a `Cache` keeps, for the paths that cannot ask which one applies. */const CACHE_LISTS = ['children', 'fills', 'shadows', 'strokes', 'effects',- 'textRanges', 'masks', 'keyframeTracks', 'keyframes', 'animations',+ 'textRanges', 'masks', 'mattes', 'keyframeTracks', 'keyframes', 'animations',] as const;/**diff --git a/packages/runtime/src/constants.ts b/packages/runtime/src/constants.tsindex 917d41b1..670df7ec 100644--- a/packages/runtime/src/constants.ts+++ b/packages/runtime/src/constants.ts@@ -89,6 +89,7 @@ export enum EffectType {INVERT,SATURATE,SEPIA,+ OPACITY,}export enum MotionType {@@ -175,4 +176,5 @@ export enum ToolType {RECT,TEXT,TEXT_EDIT,+ OBJECT_MASK,}diff --git a/packages/runtime/src/traits/derived.ts b/packages/runtime/src/traits/derived.tsindex 7e02ba4e..ed0cb219 100644--- a/packages/runtime/src/traits/derived.ts+++ b/packages/runtime/src/traits/derived.ts@@ -56,6 +56,7 @@ export const Cache = trait({effects: () => [] as Entity[],textRanges: () => [] as Entity[],masks: () => [] as Entity[],+ mattes: () => [] as Entity[],keyframeTracks: () => [] as Entity[],keyframes: () => [] as Entity[],animations: () => [] as Entity[],diff --git a/packages/runtime/src/traits/style.ts b/packages/runtime/src/traits/style.tsindex 126869af..171ac0de 100644--- a/packages/runtime/src/traits/style.ts+++ b/packages/runtime/src/traits/style.ts@@ -37,6 +37,12 @@ export const Effect = trait({value: 0,});+// A `<mask>`: a matte limiting the effect holding it, ChildOf the effect.+export const Matte = trait({+ offset: 0,+ inverted: false,+});+// Single gradient stop: its position along the gradient (0-1). Its color and// opacity are the entity's Color and Opacity. Each stop is its own entity, ChildOf the// gradient fill. Stop count is fixed for the lifetime of the fill: to animatediff --git a/packages/runtime/src/utils/index.ts b/packages/runtime/src/utils/index.tsindex 74206988..ccd788a5 100644--- a/packages/runtime/src/utils/index.ts+++ b/packages/runtime/src/utils/index.ts@@ -6,6 +6,7 @@ export * from './generating';export * from './text';export * from './sort';export * from './stroke';+export * from './surface';export * from './text-motion';export * from './time';export * from './transition';diff --git a/packages/runtime/src/world/serialize.ts b/packages/runtime/src/world/serialize.tsindex fa6f4f95..f456a130 100644--- a/packages/runtime/src/world/serialize.ts+++ b/packages/runtime/src/world/serialize.ts@@ -9,7 +9,7 @@ import {Caption,Position, Offset, Rotation, Scale, UniformScale, Anchor, Skew, Size, Flip,Constraint, KeepAspectRatio,- Opacity, BlendMode, Color, CornerRadius, MixedCornerRadius, Blur, ScaleMode, Effect,+ Opacity, BlendMode, Color, CornerRadius, MixedCornerRadius, Blur, ScaleMode, Effect, Matte,ColorStop, StrokeStyle, Shader,Chars, TextStyle,Delay, Trim, PlaybackRate, SourceFrameRate,@@ -41,6 +41,10 @@ export interface EntityRecord {type?: number;value?: number;};+ Matte?: {+ offset?: number;+ inverted?: boolean;+ };Caption?: {type?: number;colors?: number[];@@ -199,6 +203,10 @@ export function serializeEntity(entity: Entity): EntityRecord {const effect = entity.get(Effect)!;record.Effect = { type: effect.type, value: effect.value };}+ if (entity.has(Matte)) {+ const matte = entity.get(Matte)!;+ record.Matte = { offset: matte.offset, inverted: matte.inverted };+ }if (entity.has(Shadow)) {record.Shadow = {};}@@ -441,6 +449,10 @@ export function deserializeEntity(entity: Entity, e: Partial<EntityRecord>): voientity.add(Effect);entity.set(Effect, defined(e.Effect));}+ if (e.Matte !== undefined) {+ entity.add(Matte);+ entity.set(Matte, defined(e.Matte));+ }if (e.Shadow !== undefined) {entity.add(Shadow);}