The VS Code of video editing, built for agents.
TypeScript
Update transform handling in interactions and shortcuts for improved editing
konstantin-paulus6b4e7033a22e
diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.tsindex e5e1772b..a02c17be 100644--- a/apps/desktop/src/main.ts+++ b/apps/desktop/src/main.ts@@ -40,7 +40,7 @@ import type { LogEntry } from "@diffusionstudio/cli/protocol";const DEV_URL = "http://localhost:5173";const AUTH_PROTOCOL = "diffusion";-const MACOS_CORNER_RADIUS = 16;+const MACOS_CORNER_RADIUS = 18;const MACOS_BACKDROP = { blur: 80, red: 0.07, green: 0.07, blue: 0.07, alpha: 0.9 };app.setName("Diffusion Studio");diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.tsindex a85134e6..804e7e03 100644--- a/apps/desktop/src/projects.ts+++ b/apps/desktop/src/projects.ts@@ -160,12 +160,39 @@ function packageName(name: string): string {const JSX_VERSION = "^0.1.0";const SOLID_VERSION = "^1.9.10";+/**+ * The dapi surface as npm scripts: the CLI is how a project is inspected and+ * cut, so its commands belong in the record of the project they act on —+ * `npm run` prints the menu, `npm run <name> -- <args>` runs one. Named after+ * the command rather than its path (`grab`, not `media:grab`): the `media`+ * subcommands have no top-level namesakes to collide with.+ */+const SCRIPTS: Record<string, string> = {+ context: "dapi context",+ capture: "dapi capture",+ probe: "dapi media probe",+ transcribe: "dapi media transcribe",+ grab: "dapi media grab",+ filmstrip: "dapi media filmstrip",+ waveform: "dapi media waveform",+ listen: "dapi media listen",+ models: "dapi models",+ voices: "dapi voices",+ fonts: "dapi fonts",+ whoami: "dapi whoami",+ logs: "dapi logs",+ screenshot: "dapi screenshot",+ report: "dapi report",+ fetch: "dapi fetch",+};+const packageJson = (name: string, displayName: string): PackageJson => ({name: packageName(name),displayName,private: true,type: "module",main: "index.tsx",+ scripts: { ...SCRIPTS },devDependencies: {"@diffusionstudio/jsx": JSX_VERSION,"solid-js": SOLID_VERSION,@@ -201,6 +228,111 @@ const STARTER = `export default function Project() {}`;+/**+ * The project's own README: what the folder holds, how the composition is+ * written, and what can be run against it. Written once like the rest of the+ * scaffold, so from then on it is the author's file to say what this project+ * is.+ */+const readme = (displayName: string): string => `# ${displayName}++A Diffusion Studio project: a video composition authored as code. The folder is+a plain npm package whose entry file is a [Solid](https://www.solidjs.com)+component; the app compiles it and renders every element into an editable node+on the canvas.++## Structure++| Path | What it is |+| ---- | ---------- |+| \`index.tsx\` | The entry. Its default export renders the composition. |+| \`package.json\` | The project record: \`displayName\` (the name shown in the app), \`main\` (the entry), \`diffusion\` (how each scene is exported), and the dapi commands as scripts. |+| \`tsconfig.json\` | Types for the composition tags, through \`jsxImportSource\`. |+| \`assets.yml\` | The asset library: for every asset its library path, where its bytes are, and what it was found to be. Written by the app; hand edits are read on the next load. |+| \`assets/\` | Files the app produced itself, generations under \`assets/generated/\`. Media imported from elsewhere on disk is linked where it lies, never copied. |+| \`cache/\` | Derived data (thumbnails, waveforms). Disposable, and not checked in. |++## Authoring++The source is the document, in both directions. Saving recompiles the project+and remounts it — scenes rebuild in place, the way reloading a page does, and a+compile error leaves the last good render on the canvas. Edits made in the app+come back the other way: a dragged rect, a trimmed clip or a retyped line lands+as a prop on the element it was authored as.++\`\`\`tsx+export default function Project() {+ return (+ <stage background="#161616">+ <scene name="Intro" width={1920} height={1080} fill="black">+ <video src="b-roll/drone.mp4" start={0} end={6} width={1920} height={1080} />+ <text y={860} width={1920} textAlign="center" fontFamily="Inter" fontSize={96} start={1} end={5}>+ Hello+ <animation type="fade" duration={0.5} />+ <animation type="fade" phase="out" duration={0.5} />+ </text>+ </scene>+ </stage>+ )+}+\`\`\`++- \`<stage>\` is the root, and holds one \`<scene>\` per frame you cut in. A scene+ owns the timeline its children sit on; nothing outside a scene has a clock to+ be placed against.+- Position and size are explicit, in pixels. There is no layout pass and no CSS.+- \`start\` / \`end\` place a clip on that timeline, \`sourceIn\` / \`sourceOut\` choose+ the part of the media that plays. Times are seconds (\`1.5\`), frames (\`45f\`) or+ \`"MM:SS"\`.+- Style and motion are children: \`<animation>\` for a preset in or out,+ \`<keyframeTrack>\` with \`<keyframe>\` children for one property, \`<solidPaint>\`+ and the gradient paints, \`<stroke>\`, \`<shadow>\`, \`<effect>\`.+- \`src\` takes a library path (\`"b-roll/drone.mp4"\` — the portable form, it+ survives the file being relinked), an asset id, a URL, or an absolute path.+- Generated assets are declared rather than fetched: \`src={generate.image({ prompt })}\`,+ and \`generate.video\`, \`generate.voice\`, \`generate.audio\`. They are produced on+ mount, in dependency order.+- Solid is fully available while mounting: \`<For>\`, \`<Show>\`, \`createMemo\`, and+ \`useTicker()\` for values that follow the playhead.+- Any npm package can be imported without installing it: bare specifiers resolve+ from a CDN at runtime. Add one as a dev dependency (\`npm i -D three\`) only to+ get its types.++Types are stripped at compile time and never checked, so typecheck the project+yourself with \`npx tsc --noEmit\`.++## Commands++Every dapi command is a script here: \`npm run\` lists them, and+\`npm run <name> -- <args>\` runs one (\`npm run grab -- b-roll/drone.mp4 -c 6\`).+All of them talk to the running app, except \`fonts\` and \`fetch\`.++| Script | Command | What it does |+| ------ | ------- | ------------ |+| \`context\` | \`dapi context\` | What is open: scenes, active scene, playhead, work area, selection. |+| \`capture\` | \`dapi capture <id>\` | Render one node in isolation to labelled PNG contact sheets. |+| \`probe\` | \`dapi media probe <id\\|path>\` | Container and per-track metadata, without decoding. |+| \`transcribe\` | \`dapi media transcribe <id\\|path>\` | Timed speech transcript, word by word. |+| \`grab\` | \`dapi media grab <id\\|path>\` | Decode frames of a video to labelled PNG contact sheets. |+| \`filmstrip\` | \`dapi media filmstrip <id\\|path>\` | Thumbnail grid across a window of a video. |+| \`waveform\` | \`dapi media waveform <id\\|path>\` | Loudness over time, with the silences marked. |+| \`listen\` | \`dapi media listen <id\\|path>\` | Ask a multimodal model what is in an audio track. |+| \`models\` | \`dapi models [type]\` | Generation models and their per-model constraints. |+| \`voices\` | \`dapi voices\` | Speech voices for \`generate.voice\`. |+| \`fonts\` | \`dapi fonts\` | Local font families, valid as \`fontFamily\`. |+| \`whoami\` | \`dapi whoami\` | The signed-in account. |+| \`logs\` | \`dapi logs\` | Recent console output from the app. |+| \`screenshot\` | \`dapi screenshot\` | The whole app window as a PNG. |+| \`report\` | \`dapi report <title>\` | File a bug against the editor, with diagnostics attached. |+| \`fetch\` | \`dapi fetch <url>\` | Download a video with yt-dlp (installed separately). |++## Reference++- [JSX reference](https://github.com/diffusionstudio/editor/blob/main/reference/jsx/README.md): elements, timing, paints, generation, captions+- [CLI reference](https://github.com/diffusionstudio/editor/blob/main/reference/README.md): every command, its options and its output+- [Examples](https://github.com/diffusionstudio/editor/tree/main/examples): runnable compositions to read+`;+async function writeIfMissing(dir: string, name: string, content: string): Promise<void> {const path = join(dir, name);if (await exists(path)) return;@@ -217,7 +349,12 @@ async function ensurePackage(dir: string, name: string, displayName: string, entconst next = { ...pkg };if (typeof next.displayName !== "string") next.displayName = displayName;if (typeof next.main !== "string") next.main = entry;- if (next.displayName !== pkg.displayName || next.main !== pkg.main) await writePackage(dir, next);+ // The commands are a menu rather than a record: a project that keeps its own+ // scripts is left with them, one with none is given the dapi surface.+ if (typeof next.scripts !== "object" || next.scripts === null) next.scripts = { ...SCRIPTS };+ if (next.displayName !== pkg.displayName || next.main !== pkg.main || next.scripts !== pkg.scripts) {+ await writePackage(dir, next);+ }}/**@@ -238,6 +375,7 @@ export async function scaffold(dir: string, displayName = basename(dir)): Promisawait ensurePackage(dir, name, displayName, entry);await writeIfMissing(dir, "tsconfig.json", TSCONFIG);await writeIfMissing(dir, ".gitignore", GITIGNORE);+ await writeIfMissing(dir, "README.md", readme(displayName));}/** Creates a fresh project folder under `root`. Fails if the folder exists. */diff --git a/apps/web/src/engine/input/interactions.ts b/apps/web/src/engine/input/interactions.tsindex df9f39a9..dbd1f459 100644--- a/apps/web/src/engine/input/interactions.ts+++ b/apps/web/src/engine/input/interactions.ts@@ -30,6 +30,7 @@ import {import { Not, Or } from 'koota';import { getDocumentEditor } from '../editor';+import { syncKeyframe } from '../keyframes';import { AssetSelection, Hud, Keys, Pointer, SnapLines } from '../traits';import { updateCursor, type CursorType } from './cursor';import { mountNameInput } from '../hud/name-input';@@ -40,6 +41,7 @@ import {snapshotSnapCandidates,} from './snapping';+import type { DocumentEditor } from '../editor';import type { Entity, World } from 'koota';import type { DispatchedPointerEvent, Mat2D, Point, Quad } from '@diffusionstudio/runtime';@@ -71,6 +73,68 @@ const HANDLE_CURSOR: Record<Handle, CursorType> = {const isHandle = (id: string): id is Handle => id in HANDLE_FACTOR;+/** The transform props a gesture writes, as the JSX names them. */+type TransformProp = 'x' | 'y' | 'rotation' | 'width' | 'height';++/**+ * What the node shows for `name` right now, rounded the way a gesture writes+ * it: the animated value when it is keyframed, the authored one otherwise.+ */+function shownTransform(world: World, entity: Entity, name: TransformProp): number {+ const computed = store(world, Computed);+ const eid = entity.id();++ switch (name) {+ case 'x': return Math.round(computed.positionX[eid] ?? 0);+ case 'y': return Math.round(computed.positionY[eid] ?? 0);+ case 'rotation': return Math.round((computed.rotation[eid] ?? 0) * 100) / 100;+ case 'width': return Math.round(computed.width[eid] ?? 0);+ case 'height': return Math.round(computed.height[eid] ?? 0);+ }+}++/** One prop of a gesture's write, in the order the props are written. */+export type TransformWrite = [name: TransformProp, value: number];++/**+ * The transform props a gesture writes, as the gesture means them: the props+ * themselves, and the keyframes at the playhead for the ones that are+ * keyframed. An animated prop belongs to the motion system, which writes it+ * from the track on every tick, so a gesture that only wrote the prop would+ * be undone before the next frame is drawn — the node would sit where its+ * keyframes put it however far the pointer went (see `syncKeyframe`). Nothing+ * is keyframed by this: without a track already there, these are the plain+ * edits they look like.+ *+ * Only what the gesture actually changed reaches a track. A gesture writes+ * its whole set every frame, most of it the value already shown, and+ * keyframing those would leave a keyframe at the playhead for a drag that+ * never touched them — resizing by the bottom-right corner, say, which leaves+ * the position where it was.+ *+ * Hence the two passes. What is shown is read before anything is written,+ * because a prop reaches Computed with the rest of its trait: writing `x`+ * mirrors the authored `y` over the animated one, and `width` does the same+ * to `height` through the Size observer, so a value read after its sibling's+ * write is the wrong one to compare against. The keyframes then go before the+ * props, as in the inspector's rows: `width` and `height` reach the traits+ * through `resizeEntity`, which keeps their tracks in step itself and would+ * mint a keyframe of its own — one the document never made, and so cannot+ * write to the file — if it found none at the playhead. With the editor's+ * there already, all it does is set the value again.+ */+export function editTransform(+ world: World,+ editor: DocumentEditor,+ entity: Entity,+ writes: readonly TransformWrite[],+): void {+ const changed = writes.filter(([name, value]) => value !== shownTransform(world, entity, name));++ for (const [name, value] of changed) syncKeyframe(world, editor, entity, name, value);+ for (const [name, value] of writes) editor.editProperty(entity, name, value);+}+const keys = (world: World): Set<string> => world.get(Keys)?.held ?? new Set<string>();const resolution = (world: World): number => world.get(RenderSurface)?.resolution ?? 1;@@ -427,13 +491,16 @@ function resizeNode(world: World, entity: Entity, oldTr: Mat2D, localScale: Mat2const height = snapshot.height * (decomposed.scaleY / snapshot.scaleY);const offset = entityOffset(world, entity);- editor.editProperty(entity, 'width', Math.round(width));- editor.editProperty(entity, 'height', Math.round(height));- editor.editProperty(entity, 'x', Math.round(decomposed.x - width * anchor.x - offset.x));- editor.editProperty(entity, 'y', Math.round(decomposed.y - height * anchor.y - offset.y));+ const writes: TransformWrite[] = [+ ['width', Math.round(width)],+ ['height', Math.round(height)],+ ['x', Math.round(decomposed.x - width * anchor.x - offset.x)],+ ['y', Math.round(decomposed.y - height * anchor.y - offset.y)],+ ];+ if (writeAngles) writes.push(['rotation', Math.round(decomposed.rotation * 100) / 100]);+ editTransform(world, editor, entity, writes);if (writeAngles) {- editor.editProperty(entity, 'rotation', Math.round(decomposed.rotation * 100) / 100);// Skew has no JSX spelling, so it is written to the trait alone and// lasts until the project is rendered again.entity.add(Skew);@@ -474,8 +541,10 @@ function keepGroupPlaced(world: World, group: Entity): void {const offset = entityOffset(world, group);const editor = getDocumentEditor(world);- editor.editProperty(group, 'x', Math.round(position.e - offset.x));- editor.editProperty(group, 'y', Math.round(position.f - offset.y));+ editTransform(world, editor, group, [+ ['x', Math.round(position.e - offset.x)],+ ['y', Math.round(position.f - offset.y)],+ ]);computeLocalMatrix(world, group);}@@ -537,9 +606,11 @@ export function handleRotateInteraction(world: World, event: DispatchedPointerEv}const offset = entityOffset(world, entity);- editor.editProperty(entity, 'x', Math.round(decomposed.x - anchorX - offset.x));- editor.editProperty(entity, 'y', Math.round(decomposed.y - anchorY - offset.y));- editor.editProperty(entity, 'rotation', Math.round(decomposed.rotation * 100) / 100);+ editTransform(world, editor, entity, [+ ['x', Math.round(decomposed.x - anchorX - offset.x)],+ ['y', Math.round(decomposed.y - anchorY - offset.y)],+ ['rotation', Math.round(decomposed.rotation * 100) / 100],+ ]);}updateRotationCursor(world);@@ -637,8 +708,10 @@ export function handleMaskInteraction(world: World, event: DispatchedPointerEvenconst decomposed = decompose2D(moved);const offset = entityOffset(world, entity);- editor.editProperty(entity, 'x', Math.round(decomposed.x - anchorX - offset.x));- editor.editProperty(entity, 'y', Math.round(decomposed.y - anchorY - offset.y));+ editTransform(world, editor, entity, [+ ['x', Math.round(decomposed.x - anchorX - offset.x)],+ ['y', Math.round(decomposed.y - anchorY - offset.y)],+ ]);}}diff --git a/apps/web/src/engine/input/shortcuts.ts b/apps/web/src/engine/input/shortcuts.tsindex 4f6e10a8..7bfa2742 100644--- a/apps/web/src/engine/input/shortcuts.ts+++ b/apps/web/src/engine/input/shortcuts.ts@@ -9,12 +9,14 @@* pressed, matches what is held against the table below and runs the action.*/-import { getActiveEntity, getEntityChildren, getParentEntity, getSelection, isGroupLike, Position, Selected } from '@diffusionstudio/runtime';+import { Computed, getActiveEntity, getEntityChildren, getParentEntity, getSelection, isGroupLike, Position, Selected, store } from '@diffusionstudio/runtime';import { getDocumentEditor } from '../editor';import { splitAtPlayhead } from '../split';import { Keys } from '../traits';+import { editTransform } from './interactions';+import type { TransformWrite } from './interactions';import type { World } from 'koota';interface Shortcut {@@ -85,16 +87,23 @@ const NUDGE_FAST = 10;/*** Moves every selected node by `dx`, `dy` in its own parent's space, the same* `x`/`y` a drag writes, so a nudge is a drag of a known distance without the- * snapping.+ * snapping. Measured from where the node is drawn rather than from the prop,+ * and written to the position track as well as the prop, so nudging an+ * animated node moves it from where its keyframes put it (a drag does the+ * same; see `editTransform`).*/export function nudgeSelection(world: World, dx: number, dy: number): void {const editor = getDocumentEditor(world);+ const computed = store(world, Computed);for (const entity of getSelection(world)) {- const position = entity.get(Position);- if (!position) continue;- if (dx) editor.editProperty(entity, 'x', position.x + dx);- if (dy) editor.editProperty(entity, 'y', position.y + dy);+ if (!entity.has(Position)) continue;+ const eid = entity.id();++ const writes: TransformWrite[] = [];+ if (dx) writes.push(['x', Math.round((computed.positionX[eid] ?? 0) + dx)]);+ if (dy) writes.push(['y', Math.round((computed.positionY[eid] ?? 0) + dy)]);+ editTransform(world, editor, entity, writes);}}diff --git a/packages/runtime/src/actions/resize.ts b/packages/runtime/src/actions/resize.tsindex f2540202..c81dc327 100644--- a/packages/runtime/src/actions/resize.ts+++ b/packages/runtime/src/actions/resize.ts@@ -11,7 +11,6 @@ import {KeepAspectRatio, Sequential, Computed,} from '../traits';import { getParentNode } from '../queries/hierarchy';-import { syncKeyframeTrack } from './keyframe';import type { Entity, World } from 'koota';@@ -19,9 +18,17 @@ type ResizeParams = { width?: number; height?: number };/*** Resize an entity. Enforces aspect ratio, writes the final Size, propagates- * Computed size down the subtree, reapplies sibling constraints, and syncs- * the width/height keyframe tracks. Group/Sequential entities don't own a Size- * so calling this on one strips the trait instead.+ * Computed size down the subtree and reapplies sibling constraints.+ * Group/Sequential entities don't own a Size so calling this on one strips+ * the trait instead.+ *+ * Keyframes are not this function's business, for all that `width` and+ * `height` can carry a track: a keyframe belongs to the document that spelled+ * the track, and one minted here would exist only in the world — never+ * written back to the project. The editing surfaces that resize (the canvas+ * gesture, the inspector's layout rows) write the keyframe themselves, before+ * the size, and everything else that resizes — a reframe, a caption preset —+ * means the size alone.*/export function resizeEntity(world: World, entity: Entity, params: ResizeParams): void {if (entity.has(Group)) {@@ -52,13 +59,6 @@ export function resizeEntity(world: World, entity: Entity, params: ResizeParams)});// The Size observer propagates Computed sizes; constraints resolve here.resolveConstraintOffsets(world, entity);-- if (width !== undefined) {- syncKeyframeTrack(world, entity, 'width');- }- if (height !== undefined) {- syncKeyframeTrack(world, entity, 'height');- }}export function propagateSize(world: World, entity: Entity) {diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.tsindex 33867cf6..a5a278cd 100644--- a/packages/runtime/src/world/observers.ts+++ b/packages/runtime/src/world/observers.ts@@ -13,8 +13,9 @@// Keyframe auto-sync on authored edits deliberately did not move here: koota// change events don't say which field changed, and syncing every field of a// trait would mint spurious keyframes on untouched properties. Editing-// surfaces call syncKeyframeTrack for the property they changed (resizeEntity-// already does for width/height).+// surfaces keep the track of the property they changed in step themselves —+// in an app, through whatever owns the document, since a keyframe minted in+// the world alone would never reach the project it was authored in.import { store } from './store';import {