The VS Code of video editing, built for agents.
TypeScript
Implement scene support and refactor related components
konstantin-paulus0c268bd18483
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.tsindex a2ab375c..2d2cdf5e 100644--- a/apps/desktop/src/projects.ts+++ b/apps/desktop/src/projects.ts@@ -189,7 +189,6 @@ const TSCONFIG = `{const STARTER = `export default function Project() {return (<stage background="#161616">- <rect x={120} y={120} width={480} height={320} fill="#6366f1" /></stage>)}diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.tsindex 820be00b..721531c4 100644--- a/packages/jsx/src/index.ts+++ b/packages/jsx/src/index.ts@@ -51,6 +51,7 @@ export type {Keyframe,PatchProps,RectProps,+ SceneProps,SequenceProps,ShaderPaintProps,SolidPaintProps,diff --git a/packages/jsx/src/jsx-runtime.ts b/packages/jsx/src/jsx-runtime.tsindex 0b92b73a..1cc734a1 100644--- a/packages/jsx/src/jsx-runtime.ts+++ b/packages/jsx/src/jsx-runtime.ts@@ -30,6 +30,7 @@ import type {HtmlProps,ImageProps,RectProps,+ SceneProps,SequenceProps,ShaderPaintProps,SolidPaintProps,@@ -68,6 +69,7 @@ export declare namespace JSX {export interface IntrinsicElements extends HtmlElementTags, SvgElementTags {img: ImgTag;stage: StageProps & SourceProps;+ scene: SceneProps & SourceProps;group: GroupProps & SourceProps;rect: (RectProps & SourceProps) | SolidJSX.SVGElementTags["rect"];video: VideoProps & SourceProps;diff --git a/packages/jsx/src/source.ts b/packages/jsx/src/source.tsindex f6e16fc9..be50778c 100644--- a/packages/jsx/src/source.ts+++ b/packages/jsx/src/source.ts@@ -16,16 +16,13 @@/*** Attribute an element may carry to name itself. An id outlives edits that* renumber the file, so it — not the element's position — is what an entity- * holds once its element has one.+ * holds once its element has one. It is also how elements point at each other+ * within a render: `syncTo` names the id of the clip it aligns against.** Yours to write: an id is any string, and `id="hero"` is worth more to read* than anything generated. The editor only stamps elements that have none, and* renaming one by hand is safe. Stripped at compile time, so it never reaches* a host as a prop; what survives is `SOURCE_ATTR`, which carries it.- *- * Not to be confused with the `key` prop, which is a within-render label that- * `syncTo` references — a way for elements to point at each other, not a name- * that survives an edit.*/export const ID_ATTR = "id";@@ -54,6 +51,7 @@ export type PropValue = number | string | boolean | null | PropValue[] | { [key:*/export const COMPOSITION_TAGS = ["stage",+ "scene","group","rect","video",diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 4dd81d07..54c46b2a 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -116,20 +116,6 @@ export type CaptionPreset =* so JSX and any patch API can't drift.*/export type PatchProps = {- /**- * A within-render label so another node can reference this one: `syncTo`- * names the `key` of the clip it aligns against. Not a mount mechanism, and- * invalid on a scene root, which is identified by `scene`.- */- key?: string;- /**- * Promotes this element to a scene (mount root), and is the root's stable- * identity across mounts: re-mounting replaces the scene carrying the same- * `scene` value, or creates it if absent. Only valid on `<rect>`; a scene- * root cannot also carry placement or timing props (`x`/`y`/`start`/`end`/- * `transition`/`animations`) or a separate `key`.- */- scene?: string;/** Human-readable node name. */name?: string;/** Position relative to the parent, px. Defaults to 0. Animatable. */@@ -160,7 +146,7 @@ export type PatchProps = {/** Source out point: where playback ends within the source. Defaults to the natural end. Alternative to `end`. */sourceOut?: Time;/**- * Key of another element carrying an audio track. Derives the timeline+ * `id` of another element carrying an audio track. Derives the timeline* placement (`start`) by cross-correlating the two audio signals so the* recordings coincide on the timeline. Mutually exclusive with `start`.*/@@ -242,7 +228,7 @@ type TimingProps = Pick<PatchProps, "start" | "end" | "sourceIn" | "sourceOut">;type CommonProps = TimingProps &Pick<PatchProps,- | "key" | "scene" | "name" | "x" | "y" | "offsetX" | "offsetY" | "width" | "height" | "rotation"+ | "name" | "x" | "y" | "offsetX" | "offsetY" | "width" | "height" | "rotation"| "opacity" | "cornerRadius" | "transition" | "animations">;@@ -262,7 +248,10 @@ export type SourceProps = {*/export type CameraMatrix = [a: number, b: number, c: number, d: number, e: number, f: number];-/** The infinite canvas every project renders into; only allowed as the root element. */+/**+ * The infinite canvas every project renders into; only allowed as the root+ * element, and holding `<scene>` children.+ */export type StageProps = {/** Canvas color, any CSS color. */background?: string;@@ -276,6 +265,22 @@ export type StageProps = {children?: SolidJSX.Element;};+/**+ * A scene: the clipped, playable frame a composition is made in, and the only+ * element allowed directly under `<stage>`. It clips its children to+ * `width`×`height` and owns the timeline they are placed on, so it takes no+ * timing of its own — nothing outside a scene has a clock to place it against.+ *+ * `x`/`y` are where the frame sits on the infinite canvas. That is an editor+ * concern rather than part of the composition, but it lives here for the same+ * reason `<stage>`'s `camera` does: the source is the document, so a scene+ * dragged on the canvas has nowhere else to be written back to.+ */+export type SceneProps = Required<Pick<PatchProps, "width" | "height">> &+ Pick<PatchProps, "name" | "x" | "y" | "fill"> & {+ children?: SolidJSX.Element;+ };+export type GroupProps = CommonProps & Pick<PatchProps, "fill"> & {children?: SolidJSX.Element;};diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 56aa6bf6..a22d59f8 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -4,22 +4,31 @@import {+ ActiveScene,+ appendChild,Background,- createEntity,+ ClipsContent,DEFAULT_BACKGROUND,Color,Geometry,GeometryType,getParentEntity,+ getParentNode,ItemIndex,+ Name,parseColor,+ Playback,Position,+ removeChild,resizeEntity,getEntityChildren,DocumentRoot,Root,+ Scene,Source,setCameraMatrix,+ switchActiveScene,+ Size,} from '@diffusionstudio/runtime';import { SOURCE_ATTR } from '@diffusionstudio/jsx';@@ -98,20 +107,29 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {}public createElement(tag: string): SceneNode {- if (tag === 'stage') {- return this.stage;- } else if (tag === 'rect') {- const world = this.world;- const entity = createEntity(world);- entity.add(Geometry);- entity.set(Geometry, { value: GeometryType.RECT });- entity.add(Position);- entity.set(Position, { x: 0, y: 0 });- resizeEntity(world, entity, { width: 100, height: 100 });-- return { entity };- } else {- throw new Error(`<${tag}> is not supported yet (only <stage> and <rect>).`);+ switch (tag) {+ case 'stage':+ return this.stage;+ case 'scene': {+ const entity = this.world.spawn(+ Geometry({ value: GeometryType.RECT }),+ Position({ x: 0, y: 0 }),+ Size({ width: 1920, height: 1080 }),+ Scene,+ ClipsContent,+ Playback,+ );+ return { entity };+ }+ case 'rect':+ const entity = this.world.spawn(+ Geometry({ value: GeometryType.RECT }),+ Position({ x: 0, y: 0 }),+ Size({ width: 100, height: 100 }),+ );+ return { entity };+ default:+ throw new Error(`<${tag}> is not supported yet (only <stage>, <scene> and <rect>).`);}}@@ -137,6 +155,16 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {entity.set(Source, { value });return;}+ case 'name': {+ if (typeof value !== 'string' || !value) {+ entity.remove(Name);+ return;+ }++ entity.add(Name);+ entity.set(Name, { value });+ return;+ }case 'x':case 'y': {entity.add(Position);@@ -183,16 +211,36 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {}}+ /**+ * Node handles are minted per lookup, so every comparison here is between+ * entities: two handles for the same node are never the same object.+ */public insertNode(parent: SceneNode, node: SceneNode, anchor?: SceneNode): void {- if (parent === node) return;- // createEntity already parented the node to the document (= the stage).- const siblings = this.children(parent).filter((sibling) => sibling !== node);- const at = anchor ? siblings.indexOf(anchor) : -1;- if (at === -1) siblings.push(node);- else siblings.splice(at, 0, node);+ if (parent.entity === node.entity) return;++ // Scenes own a timeline, so one inside another has no coherent reading.+ if (node.entity.has(Scene) && !parent.entity.has(DocumentRoot)) {+ throw new Error('<scene> is only allowed as a direct child of <stage>.');+ }++ if (getParentEntity(node.entity) !== parent.entity) {+ // appendChild only takes top-level entities, so a move between two+ // parents goes back through the document on the way.+ const current = getParentNode(node.entity);+ if (current !== null) removeChild(this.world, node.entity, current);+ // <stage> is the document root, which is where entities spawn.+ if (!parent.entity.has(DocumentRoot)) {+ appendChild(this.world, node.entity, parent.entity);+ }+ }++ const siblings = this.childEntities(parent).filter((sibling) => sibling !== node.entity);+ const at = anchor ? siblings.indexOf(anchor.entity) : -1;+ if (at === -1) siblings.push(node.entity);+ else siblings.splice(at, 0, node.entity);for (const [index, sibling] of siblings.entries()) {- sibling.entity.add(ItemIndex);- sibling.entity.set(ItemIndex, { value: index });+ sibling.add(ItemIndex);+ sibling.set(ItemIndex, { value: index });}}@@ -205,6 +253,10 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {node.entity.remove(Source);return;}+ // The playhead cannot be left aimed at an entity that is going away.+ if (this.world.get(ActiveScene)?.entity === node.entity) {+ switchActiveScene(this.world, null);+ }// ChildOf auto-destroys orphans, so paints go with it.node.entity.destroy();}@@ -221,8 +273,11 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {public getNextSibling(node: SceneNode): SceneNode | undefined {const parent = this.getParentNode(node);if (parent === undefined) return undefined;- const siblings = this.children(parent);- return siblings[siblings.indexOf(node) + 1];+ const siblings = this.childEntities(parent);+ const at = siblings.indexOf(node.entity);+ if (at === -1) return undefined;+ const next = siblings[at + 1];+ return next === undefined ? undefined : { entity: next };}public dispose(): void {@@ -231,7 +286,12 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {/** Children of `parent` in draw order, as handles. */private children(parent: SceneNode): SceneNode[] {- return getEntityChildren(this.world, parent.entity).map((entity) => ({ entity }));+ return this.childEntities(parent).map((entity) => ({ entity }));+ }++ /** Children of `parent` in draw order. */+ private childEntities(parent: SceneNode): Entity[] {+ return getEntityChildren(this.world, parent.entity);}}diff --git a/packages/reconciler/src/elements.ts b/packages/reconciler/src/elements.tsindex 22a38f08..631a1ee6 100644--- a/packages/reconciler/src/elements.ts+++ b/packages/reconciler/src/elements.ts@@ -27,6 +27,7 @@ import type {HtmlProps,ImageProps,RectProps,+ SceneProps,SequenceProps,ShaderPaintProps,SolidPaintProps,@@ -52,6 +53,7 @@ function hostElement<P extends object>(tag: string): (props: P) => SolidJSX.Elem};}+export const Scene = hostElement<SceneProps>("Scene");export const Group = hostElement<GroupProps>("Group");export const Rect = hostElement<RectProps>("Rect");export const Video = hostElement<VideoProps>("Video");diff --git a/packages/reconciler/src/patch.ts b/packages/reconciler/src/patch.tsindex 39e589b2..1db6001b 100644--- a/packages/reconciler/src/patch.ts+++ b/packages/reconciler/src/patch.ts@@ -11,8 +11,6 @@ import type { PatchProps } from '@diffusionstudio/jsx';* `PatchProps` grows.*/export const PATCH_PROP_KEYS = Object.keys({- key: true,- scene: true,name: true,x: true,y: true,diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.mdindex 11a3faf5..ab81d7e0 100644--- a/packages/runtime/MIGRATION.md+++ b/packages/runtime/MIGRATION.md@@ -108,7 +108,18 @@ apps/web now depends on `@diffusionstudio/runtime` and `@diffusionstudio/koota-s- **Engine shell** (`apps/web/src/engine/`, not in this package — it's browser/DOM-coupled, which this package deliberately isn't): `Engine` (`create-engine.ts`) is a class wrapping `createRuntimeWorld` + `runSystems`; it owns canvas mount/resize/DPR, an AudioContext (created or injected, closed on `dispose()` only if it created it), and its own `requestAnimationFrame` loop (`start`/`stop`, plus a public `step(timestamp)` escape hatch for a future host to drive the tick itself and interleave app-side input/HUD systems around it). `context.tsx` (`EngineProvider`/`useEngineContext`) composes koota-solid's `WorldProvider` around `engine.world`, so trait state is read via `useTrait`/`useQuery` directly against the world, not funneled through a context bag. `canvas.tsx` (`EngineCanvas`) is the reusable mount/resize/start component. `hooks/` has a few illustrative per-trait wrappers so far: `useCamera`, `useActiveScene`, `useRunning`.- **Wired into the live app**: `KootaEngineProvider` (the new `EngineProvider`, aliased) is mounted in `pages/home.tsx` at the same level and lifetime as the old bitecs `EngineProvider` (both keyed on `projectId`), so anything under `EditorPage` can reach the koota world as more gets migrated. In `components/canvas/canvas.tsx`, the koota `EngineCanvas` now occupies the visible `z-1` canvas slot. The old bitecs `<canvas id="engine-canvas">` stays mounted at the same DOM id (so its `EngineProvider`'s `init()` still finds it and keeps initializing, and everything still on `useEngine()`/`useECS()` — Toolbar, drag/drop, sidebars, timeline — keeps working) but is `opacity-0 pointer-events-none`: no longer visible or interactive. Drag/drop handlers moved from the canvas element to its wrapping div so `pointer-events-none` didn't silently kill them.-- **Projects on disk (desktop only)**: `apps/web/src/projects/` loads real content into the koota world. A project is a folder under a user-picked root (persisted in `dspro.ui.projects.root`) holding an `index.tsx` whose default export renders `<stage>`; the electron main process (`apps/desktop/src/projects.ts`) scans the root, scaffolds new folders (`index.tsx`, `package.json`, `tsconfig.json`, generated `jsx.d.ts`), bundles with esbuild + babel-preset-solid (universal, moduleName `@diffusionstudio/jsx`) and watches for changes. The new `@diffusionstudio/reconciler` package (`packages/reconciler`, modeled on ds-next's `src/reconciler/`) evaluates the CJS bundle against the app's solid-js and `@diffusionstudio/jsx`, and its `RuntimeDocument` implements `ProjectDocument` over the world (`mount(code, world)` wraps both): `<stage>` = the infinite canvas, i.e. the document root itself (`background` sets the Background world trait; no size, no camera fitting), `<rect>` = top-level RECT entity (x/y/width/height; `fill` is the entity's own `Color` trait — an intrinsic solid fill rendered first, beneath any Paint children — so a filled rect needs no sub-entities). Host nodes wrap entities in objects because Solid's universal renderer treats numbers as text. `useProject(name)` (`apps/web/src/projects/use-project.ts`, called from `pages/editor.tsx`) compiles → renders → re-renders on disk changes. Routes: `/` is the full-screen dashboard (`pages/home.tsx`, no longer a dialog), `/projects/*name` is the editor with the project folder `name` loaded (`useProjectId()` reads it from the route); the old default project id is gone, so there is no editor without a project. Only those two tags/five props so far; no write-back from canvas edits to source yet.+- **Projects on disk (desktop only)**: `apps/web/src/projects/` loads real content into the koota world. A project is a folder under a user-picked root (persisted in `dspro.ui.projects.root`) holding an `index.tsx` whose default export renders `<stage>`; the electron main process (`apps/desktop/src/projects.ts`) scans the root, scaffolds new folders (`index.tsx`, `package.json`, `tsconfig.json`, generated `jsx.d.ts`), bundles with esbuild + babel-preset-solid (universal, moduleName `@diffusionstudio/jsx`) and watches for changes. The new `@diffusionstudio/reconciler` package (`packages/reconciler`, modeled on ds-next's `src/reconciler/`) evaluates the CJS bundle against the app's solid-js and `@diffusionstudio/jsx`, and its `RuntimeDocument` implements `ProjectDocument` over the world (`mount(code, world)` wraps both): `<stage>` = the infinite canvas, i.e. the document root itself (`background` sets the Background world trait; no size, no camera fitting), `<scene>` = a top-level clipped, playable frame (the `frameSelection` recipe: RECT geometry + `Scene`/`ClipsContent`/`Playback`, minus the solid fill child, since `fill` is the entity's own trait), `<rect>` = a RECT entity (x/y/width/height; `fill` is the entity's own `Color` trait — an intrinsic solid fill rendered first, beneath any Paint children — so a filled rect needs no sub-entities). Host nodes wrap entities in objects because Solid's universal renderer treats numbers as text. `useProject(name)` (`apps/web/src/projects/use-project.ts`, called from `pages/editor.tsx`) compiles → renders → re-renders on disk changes. Routes: `/` is the full-screen dashboard (`pages/home.tsx`, no longer a dialog), `/projects/*name` is the editor with the project folder `name` loaded (`useProjectId()` reads it from the route); the old default project id is gone, so there is no editor without a project. Only those three tags/seven props so far; no write-back from canvas edits to source yet.++ **The `<scene>` slice.** The `scene="id"` prop that promoted a `<rect>` is gone (dropped from `PatchProps`, `CommonProps`, and the `PATCH_PROP_KEYS` allowlist): it existed to give a mount root an identity across partial mounts, and with `<stage>` standing for the whole document and `id`/`SOURCE_ATTR` carrying identity, nothing was left for it to do. What the tag added beyond the entity recipe:+ - **`x`/`y` on a scene are authored.** The old reference model made canvas placement an editor concern with no JSX spelling. That cannot hold once the source is the document: a scene dragged on the canvas has nowhere to be written back to. Same reasoning as `<stage camera>`.+ - **`insertNode` re-parents.** It previously only rewrote `ItemIndex`, so every element landed under the document root regardless of nesting. Harmless while `<stage>`/`<rect>` were the whole vocabulary; fatal for a frame that has to contain its children. A move between two non-root parents goes back through the document, because `appendChild` asserts top-level.+ - **Ordering compares entities, not handles.** `children()` mints a fresh `{ entity }` per lookup, so the old `siblings.indexOf(node)` in `insertNode`/`getNextSibling` never matched: anchors were ignored (every insert appended) and `getNextSibling` always returned the first child. Only reachable through `<Show>`/`<For>` reordering, which nothing exercised yet.+ - **`ActiveScene`**: the first scene a render produces claims it if no live scene holds it; `removeNode` clears it when the holder goes. The timeline has to point somewhere, and which scene is being worked on afterwards is the editor's state, not the document's.+ - Nested `<scene>` throws (the runtime forbids it); `name` joined the handled props.++ **The `key` prop went with it**, for the same reason: it was a within-render label letting elements point at each other (`syncTo` named a clip's `key`), which is the job `id` already does, and unlike `key` an id survives an edit that renumbers the file. So `syncTo` now names an `id`. Dropped from `PatchProps`, `CommonProps`, and `PATCH_PROP_KEYS`. The runtime's `Key` trait is untouched and unrelated: it is the ECS-side stable identity that clone/paste external refs key off (entity ids do not survive delete/undo), and nothing on the JSX path ever wrote it.++ **Left stale on purpose** (deferred to a docs/examples pass): all eight `examples/*.tsx` still open with `<rect scene="...">` and no `<stage>`, so `tsc -p examples --noEmit` fails on them; `reference/jsx/{scene,roots}.md` still describes the pre-`<stage>` model where a mount grafts roots into an existing document; and `reference/jsx/{audio-sync,video,audio,elements}.md` still document `key` as the `syncTo` target. Deliberately not half-migrated: `reference/jsx` is stale as a whole, and patching the `key` rows alone would hide that.## Remaining work (suggested order)