Skip to content

The VS Code of video editing, built for agents.

TypeScript

Update transform handling in interactions and shortcuts for improved editing

konstantin-paulus6b4e7033a22e
diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts
index e5e1772b..a02c17be 100644
--- a/apps/desktop/src/main.ts
+++ b/apps/desktop/src/main.ts
@@ -40,7 +40,7 @@ import type { LogEntry } from "@diffusionstudio/cli/protocol";
const DEV_URL = "http://localhost:5173";
const AUTH_PROTOCOL = "diffusion";
-const MACOS_CORNER_RADIUS = 16;
+const MACOS_CORNER_RADIUS = 18;
const MACOS_BACKDROP = { blur: 80, red: 0.07, green: 0.07, blue: 0.07, alpha: 0.9 };
app.setName("Diffusion Studio");
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.ts
index a85134e6..804e7e03 100644
--- a/apps/desktop/src/projects.ts
+++ b/apps/desktop/src/projects.ts
@@ -160,12 +160,39 @@ function packageName(name: string): string {
const JSX_VERSION = "^0.1.0";
const SOLID_VERSION = "^1.9.10";
+/**
+ * The dapi surface as npm scripts: the CLI is how a project is inspected and
+ * cut, so its commands belong in the record of the project they act on —
+ * `npm run` prints the menu, `npm run <name> -- <args>` runs one. Named after
+ * the command rather than its path (`grab`, not `media:grab`): the `media`
+ * subcommands have no top-level namesakes to collide with.
+ */
+const SCRIPTS: Record<string, string> = {
+ context: "dapi context",
+ capture: "dapi capture",
+ probe: "dapi media probe",
+ transcribe: "dapi media transcribe",
+ grab: "dapi media grab",
+ filmstrip: "dapi media filmstrip",
+ waveform: "dapi media waveform",
+ listen: "dapi media listen",
+ models: "dapi models",
+ voices: "dapi voices",
+ fonts: "dapi fonts",
+ whoami: "dapi whoami",
+ logs: "dapi logs",
+ screenshot: "dapi screenshot",
+ report: "dapi report",
+ fetch: "dapi fetch",
+};
+
const packageJson = (name: string, displayName: string): PackageJson => ({
name: packageName(name),
displayName,
private: true,
type: "module",
main: "index.tsx",
+ scripts: { ...SCRIPTS },
devDependencies: {
"@diffusionstudio/jsx": JSX_VERSION,
"solid-js": SOLID_VERSION,
@@ -201,6 +228,111 @@ const STARTER = `export default function Project() {
}
`;
+/**
+ * The project's own README: what the folder holds, how the composition is
+ * written, and what can be run against it. Written once like the rest of the
+ * scaffold, so from then on it is the author's file to say what this project
+ * is.
+ */
+const readme = (displayName: string): string => `# ${displayName}
+
+A Diffusion Studio project: a video composition authored as code. The folder is
+a plain npm package whose entry file is a [Solid](https://www.solidjs.com)
+component; the app compiles it and renders every element into an editable node
+on the canvas.
+
+## Structure
+
+| Path | What it is |
+| ---- | ---------- |
+| \`index.tsx\` | The entry. Its default export renders the composition. |
+| \`package.json\` | The project record: \`displayName\` (the name shown in the app), \`main\` (the entry), \`diffusion\` (how each scene is exported), and the dapi commands as scripts. |
+| \`tsconfig.json\` | Types for the composition tags, through \`jsxImportSource\`. |
+| \`assets.yml\` | The asset library: for every asset its library path, where its bytes are, and what it was found to be. Written by the app; hand edits are read on the next load. |
+| \`assets/\` | Files the app produced itself, generations under \`assets/generated/\`. Media imported from elsewhere on disk is linked where it lies, never copied. |
+| \`cache/\` | Derived data (thumbnails, waveforms). Disposable, and not checked in. |
+
+## Authoring
+
+The source is the document, in both directions. Saving recompiles the project
+and remounts it — scenes rebuild in place, the way reloading a page does, and a
+compile error leaves the last good render on the canvas. Edits made in the app
+come back the other way: a dragged rect, a trimmed clip or a retyped line lands
+as a prop on the element it was authored as.
+
+\`\`\`tsx
+export default function Project() {
+ return (
+ <stage background="#161616">
+ <scene name="Intro" width={1920} height={1080} fill="black">
+ <video src="b-roll/drone.mp4" start={0} end={6} width={1920} height={1080} />
+ <text y={860} width={1920} textAlign="center" fontFamily="Inter" fontSize={96} start={1} end={5}>
+ Hello
+ <animation type="fade" duration={0.5} />
+ <animation type="fade" phase="out" duration={0.5} />
+ </text>
+ </scene>
+ </stage>
+ )
+}
+\`\`\`
+
+- \`<stage>\` is the root, and holds one \`<scene>\` per frame you cut in. A scene
+ owns the timeline its children sit on; nothing outside a scene has a clock to
+ be placed against.
+- Position and size are explicit, in pixels. There is no layout pass and no CSS.
+- \`start\` / \`end\` place a clip on that timeline, \`sourceIn\` / \`sourceOut\` choose
+ the part of the media that plays. Times are seconds (\`1.5\`), frames (\`45f\`) or
+ \`"MM:SS"\`.
+- Style and motion are children: \`<animation>\` for a preset in or out,
+ \`<keyframeTrack>\` with \`<keyframe>\` children for one property, \`<solidPaint>\`
+ and the gradient paints, \`<stroke>\`, \`<shadow>\`, \`<effect>\`.
+- \`src\` takes a library path (\`"b-roll/drone.mp4"\` — the portable form, it
+ survives the file being relinked), an asset id, a URL, or an absolute path.
+- Generated assets are declared rather than fetched: \`src={generate.image({ prompt })}\`,
+ and \`generate.video\`, \`generate.voice\`, \`generate.audio\`. They are produced on
+ mount, in dependency order.
+- Solid is fully available while mounting: \`<For>\`, \`<Show>\`, \`createMemo\`, and
+ \`useTicker()\` for values that follow the playhead.
+- Any npm package can be imported without installing it: bare specifiers resolve
+ from a CDN at runtime. Add one as a dev dependency (\`npm i -D three\`) only to
+ get its types.
+
+Types are stripped at compile time and never checked, so typecheck the project
+yourself with \`npx tsc --noEmit\`.
+
+## Commands
+
+Every dapi command is a script here: \`npm run\` lists them, and
+\`npm run <name> -- <args>\` runs one (\`npm run grab -- b-roll/drone.mp4 -c 6\`).
+All of them talk to the running app, except \`fonts\` and \`fetch\`.
+
+| Script | Command | What it does |
+| ------ | ------- | ------------ |
+| \`context\` | \`dapi context\` | What is open: scenes, active scene, playhead, work area, selection. |
+| \`capture\` | \`dapi capture <id>\` | Render one node in isolation to labelled PNG contact sheets. |
+| \`probe\` | \`dapi media probe <id\\|path>\` | Container and per-track metadata, without decoding. |
+| \`transcribe\` | \`dapi media transcribe <id\\|path>\` | Timed speech transcript, word by word. |
+| \`grab\` | \`dapi media grab <id\\|path>\` | Decode frames of a video to labelled PNG contact sheets. |
+| \`filmstrip\` | \`dapi media filmstrip <id\\|path>\` | Thumbnail grid across a window of a video. |
+| \`waveform\` | \`dapi media waveform <id\\|path>\` | Loudness over time, with the silences marked. |
+| \`listen\` | \`dapi media listen <id\\|path>\` | Ask a multimodal model what is in an audio track. |
+| \`models\` | \`dapi models [type]\` | Generation models and their per-model constraints. |
+| \`voices\` | \`dapi voices\` | Speech voices for \`generate.voice\`. |
+| \`fonts\` | \`dapi fonts\` | Local font families, valid as \`fontFamily\`. |
+| \`whoami\` | \`dapi whoami\` | The signed-in account. |
+| \`logs\` | \`dapi logs\` | Recent console output from the app. |
+| \`screenshot\` | \`dapi screenshot\` | The whole app window as a PNG. |
+| \`report\` | \`dapi report <title>\` | File a bug against the editor, with diagnostics attached. |
+| \`fetch\` | \`dapi fetch <url>\` | Download a video with yt-dlp (installed separately). |
+
+## Reference
+
+- [JSX reference](https://github.com/diffusionstudio/editor/blob/main/reference/jsx/README.md): elements, timing, paints, generation, captions
+- [CLI reference](https://github.com/diffusionstudio/editor/blob/main/reference/README.md): every command, its options and its output
+- [Examples](https://github.com/diffusionstudio/editor/tree/main/examples): runnable compositions to read
+`;
+
async function writeIfMissing(dir: string, name: string, content: string): Promise<void> {
const path = join(dir, name);
if (await exists(path)) return;
@@ -217,7 +349,12 @@ async function ensurePackage(dir: string, name: string, displayName: string, ent
const next = { ...pkg };
if (typeof next.displayName !== "string") next.displayName = displayName;
if (typeof next.main !== "string") next.main = entry;
- if (next.displayName !== pkg.displayName || next.main !== pkg.main) await writePackage(dir, next);
+ // The commands are a menu rather than a record: a project that keeps its own
+ // scripts is left with them, one with none is given the dapi surface.
+ if (typeof next.scripts !== "object" || next.scripts === null) next.scripts = { ...SCRIPTS };
+ if (next.displayName !== pkg.displayName || next.main !== pkg.main || next.scripts !== pkg.scripts) {
+ await writePackage(dir, next);
+ }
}
/**
@@ -238,6 +375,7 @@ export async function scaffold(dir: string, displayName = basename(dir)): Promis
await ensurePackage(dir, name, displayName, entry);
await writeIfMissing(dir, "tsconfig.json", TSCONFIG);
await writeIfMissing(dir, ".gitignore", GITIGNORE);
+ await writeIfMissing(dir, "README.md", readme(displayName));
}
/** Creates a fresh project folder under `root`. Fails if the folder exists. */
diff --git a/apps/web/src/engine/input/interactions.ts b/apps/web/src/engine/input/interactions.ts
index df9f39a9..dbd1f459 100644
--- a/apps/web/src/engine/input/interactions.ts
+++ b/apps/web/src/engine/input/interactions.ts
@@ -30,6 +30,7 @@ import {
import { Not, Or } from 'koota';
import { getDocumentEditor } from '../editor';
+import { syncKeyframe } from '../keyframes';
import { AssetSelection, Hud, Keys, Pointer, SnapLines } from '../traits';
import { updateCursor, type CursorType } from './cursor';
import { mountNameInput } from '../hud/name-input';
@@ -40,6 +41,7 @@ import {
snapshotSnapCandidates,
} from './snapping';
+import type { DocumentEditor } from '../editor';
import type { Entity, World } from 'koota';
import type { DispatchedPointerEvent, Mat2D, Point, Quad } from '@diffusionstudio/runtime';
@@ -71,6 +73,68 @@ const HANDLE_CURSOR: Record<Handle, CursorType> = {
const isHandle = (id: string): id is Handle => id in HANDLE_FACTOR;
+/** The transform props a gesture writes, as the JSX names them. */
+type TransformProp = 'x' | 'y' | 'rotation' | 'width' | 'height';
+
+/**
+ * What the node shows for `name` right now, rounded the way a gesture writes
+ * it: the animated value when it is keyframed, the authored one otherwise.
+ */
+function shownTransform(world: World, entity: Entity, name: TransformProp): number {
+ const computed = store(world, Computed);
+ const eid = entity.id();
+
+ switch (name) {
+ case 'x': return Math.round(computed.positionX[eid] ?? 0);
+ case 'y': return Math.round(computed.positionY[eid] ?? 0);
+ case 'rotation': return Math.round((computed.rotation[eid] ?? 0) * 100) / 100;
+ case 'width': return Math.round(computed.width[eid] ?? 0);
+ case 'height': return Math.round(computed.height[eid] ?? 0);
+ }
+}
+
+/** One prop of a gesture's write, in the order the props are written. */
+export type TransformWrite = [name: TransformProp, value: number];
+
+/**
+ * The transform props a gesture writes, as the gesture means them: the props
+ * themselves, and the keyframes at the playhead for the ones that are
+ * keyframed. An animated prop belongs to the motion system, which writes it
+ * from the track on every tick, so a gesture that only wrote the prop would
+ * be undone before the next frame is drawn — the node would sit where its
+ * keyframes put it however far the pointer went (see `syncKeyframe`). Nothing
+ * is keyframed by this: without a track already there, these are the plain
+ * edits they look like.
+ *
+ * Only what the gesture actually changed reaches a track. A gesture writes
+ * its whole set every frame, most of it the value already shown, and
+ * keyframing those would leave a keyframe at the playhead for a drag that
+ * never touched them — resizing by the bottom-right corner, say, which leaves
+ * the position where it was.
+ *
+ * Hence the two passes. What is shown is read before anything is written,
+ * because a prop reaches Computed with the rest of its trait: writing `x`
+ * mirrors the authored `y` over the animated one, and `width` does the same
+ * to `height` through the Size observer, so a value read after its sibling's
+ * write is the wrong one to compare against. The keyframes then go before the
+ * props, as in the inspector's rows: `width` and `height` reach the traits
+ * through `resizeEntity`, which keeps their tracks in step itself and would
+ * mint a keyframe of its own — one the document never made, and so cannot
+ * write to the file — if it found none at the playhead. With the editor's
+ * there already, all it does is set the value again.
+ */
+export function editTransform(
+ world: World,
+ editor: DocumentEditor,
+ entity: Entity,
+ writes: readonly TransformWrite[],
+): void {
+ const changed = writes.filter(([name, value]) => value !== shownTransform(world, entity, name));
+
+ for (const [name, value] of changed) syncKeyframe(world, editor, entity, name, value);
+ for (const [name, value] of writes) editor.editProperty(entity, name, value);
+}
+
const keys = (world: World): Set<string> => world.get(Keys)?.held ?? new Set<string>();
const resolution = (world: World): number => world.get(RenderSurface)?.resolution ?? 1;
@@ -427,13 +491,16 @@ function resizeNode(world: World, entity: Entity, oldTr: Mat2D, localScale: Mat2
const height = snapshot.height * (decomposed.scaleY / snapshot.scaleY);
const offset = entityOffset(world, entity);
- editor.editProperty(entity, 'width', Math.round(width));
- editor.editProperty(entity, 'height', Math.round(height));
- editor.editProperty(entity, 'x', Math.round(decomposed.x - width * anchor.x - offset.x));
- editor.editProperty(entity, 'y', Math.round(decomposed.y - height * anchor.y - offset.y));
+ const writes: TransformWrite[] = [
+ ['width', Math.round(width)],
+ ['height', Math.round(height)],
+ ['x', Math.round(decomposed.x - width * anchor.x - offset.x)],
+ ['y', Math.round(decomposed.y - height * anchor.y - offset.y)],
+ ];
+ if (writeAngles) writes.push(['rotation', Math.round(decomposed.rotation * 100) / 100]);
+ editTransform(world, editor, entity, writes);
if (writeAngles) {
- editor.editProperty(entity, 'rotation', Math.round(decomposed.rotation * 100) / 100);
// Skew has no JSX spelling, so it is written to the trait alone and
// lasts until the project is rendered again.
entity.add(Skew);
@@ -474,8 +541,10 @@ function keepGroupPlaced(world: World, group: Entity): void {
const offset = entityOffset(world, group);
const editor = getDocumentEditor(world);
- editor.editProperty(group, 'x', Math.round(position.e - offset.x));
- editor.editProperty(group, 'y', Math.round(position.f - offset.y));
+ editTransform(world, editor, group, [
+ ['x', Math.round(position.e - offset.x)],
+ ['y', Math.round(position.f - offset.y)],
+ ]);
computeLocalMatrix(world, group);
}
@@ -537,9 +606,11 @@ export function handleRotateInteraction(world: World, event: DispatchedPointerEv
}
const offset = entityOffset(world, entity);
- editor.editProperty(entity, 'x', Math.round(decomposed.x - anchorX - offset.x));
- editor.editProperty(entity, 'y', Math.round(decomposed.y - anchorY - offset.y));
- editor.editProperty(entity, 'rotation', Math.round(decomposed.rotation * 100) / 100);
+ editTransform(world, editor, entity, [
+ ['x', Math.round(decomposed.x - anchorX - offset.x)],
+ ['y', Math.round(decomposed.y - anchorY - offset.y)],
+ ['rotation', Math.round(decomposed.rotation * 100) / 100],
+ ]);
}
updateRotationCursor(world);
@@ -637,8 +708,10 @@ export function handleMaskInteraction(world: World, event: DispatchedPointerEven
const decomposed = decompose2D(moved);
const offset = entityOffset(world, entity);
- editor.editProperty(entity, 'x', Math.round(decomposed.x - anchorX - offset.x));
- editor.editProperty(entity, 'y', Math.round(decomposed.y - anchorY - offset.y));
+ editTransform(world, editor, entity, [
+ ['x', Math.round(decomposed.x - anchorX - offset.x)],
+ ['y', Math.round(decomposed.y - anchorY - offset.y)],
+ ]);
}
}
diff --git a/apps/web/src/engine/input/shortcuts.ts b/apps/web/src/engine/input/shortcuts.ts
index 4f6e10a8..7bfa2742 100644
--- a/apps/web/src/engine/input/shortcuts.ts
+++ b/apps/web/src/engine/input/shortcuts.ts
@@ -9,12 +9,14 @@
* pressed, matches what is held against the table below and runs the action.
*/
-import { getActiveEntity, getEntityChildren, getParentEntity, getSelection, isGroupLike, Position, Selected } from '@diffusionstudio/runtime';
+import { Computed, getActiveEntity, getEntityChildren, getParentEntity, getSelection, isGroupLike, Position, Selected, store } from '@diffusionstudio/runtime';
import { getDocumentEditor } from '../editor';
import { splitAtPlayhead } from '../split';
import { Keys } from '../traits';
+import { editTransform } from './interactions';
+import type { TransformWrite } from './interactions';
import type { World } from 'koota';
interface Shortcut {
@@ -85,16 +87,23 @@ const NUDGE_FAST = 10;
/**
* Moves every selected node by `dx`, `dy` in its own parent's space, the same
* `x`/`y` a drag writes, so a nudge is a drag of a known distance without the
- * snapping.
+ * snapping. Measured from where the node is drawn rather than from the prop,
+ * and written to the position track as well as the prop, so nudging an
+ * animated node moves it from where its keyframes put it (a drag does the
+ * same; see `editTransform`).
*/
export function nudgeSelection(world: World, dx: number, dy: number): void {
const editor = getDocumentEditor(world);
+ const computed = store(world, Computed);
for (const entity of getSelection(world)) {
- const position = entity.get(Position);
- if (!position) continue;
- if (dx) editor.editProperty(entity, 'x', position.x + dx);
- if (dy) editor.editProperty(entity, 'y', position.y + dy);
+ if (!entity.has(Position)) continue;
+ const eid = entity.id();
+
+ const writes: TransformWrite[] = [];
+ if (dx) writes.push(['x', Math.round((computed.positionX[eid] ?? 0) + dx)]);
+ if (dy) writes.push(['y', Math.round((computed.positionY[eid] ?? 0) + dy)]);
+ editTransform(world, editor, entity, writes);
}
}
diff --git a/packages/runtime/src/actions/resize.ts b/packages/runtime/src/actions/resize.ts
index f2540202..c81dc327 100644
--- a/packages/runtime/src/actions/resize.ts
+++ b/packages/runtime/src/actions/resize.ts
@@ -11,7 +11,6 @@ import {
KeepAspectRatio, Sequential, Computed,
} from '../traits';
import { getParentNode } from '../queries/hierarchy';
-import { syncKeyframeTrack } from './keyframe';
import type { Entity, World } from 'koota';
@@ -19,9 +18,17 @@ type ResizeParams = { width?: number; height?: number };
/**
* Resize an entity. Enforces aspect ratio, writes the final Size, propagates
- * Computed size down the subtree, reapplies sibling constraints, and syncs
- * the width/height keyframe tracks. Group/Sequential entities don't own a Size
- * so calling this on one strips the trait instead.
+ * Computed size down the subtree and reapplies sibling constraints.
+ * Group/Sequential entities don't own a Size so calling this on one strips
+ * the trait instead.
+ *
+ * Keyframes are not this function's business, for all that `width` and
+ * `height` can carry a track: a keyframe belongs to the document that spelled
+ * the track, and one minted here would exist only in the world — never
+ * written back to the project. The editing surfaces that resize (the canvas
+ * gesture, the inspector's layout rows) write the keyframe themselves, before
+ * the size, and everything else that resizes — a reframe, a caption preset —
+ * means the size alone.
*/
export function resizeEntity(world: World, entity: Entity, params: ResizeParams): void {
if (entity.has(Group)) {
@@ -52,13 +59,6 @@ export function resizeEntity(world: World, entity: Entity, params: ResizeParams)
});
// The Size observer propagates Computed sizes; constraints resolve here.
resolveConstraintOffsets(world, entity);
-
- if (width !== undefined) {
- syncKeyframeTrack(world, entity, 'width');
- }
- if (height !== undefined) {
- syncKeyframeTrack(world, entity, 'height');
- }
}
export function propagateSize(world: World, entity: Entity) {
diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.ts
index 33867cf6..a5a278cd 100644
--- a/packages/runtime/src/world/observers.ts
+++ b/packages/runtime/src/world/observers.ts
@@ -13,8 +13,9 @@
// Keyframe auto-sync on authored edits deliberately did not move here: koota
// change events don't say which field changed, and syncing every field of a
// trait would mint spurious keyframes on untouched properties. Editing
-// surfaces call syncKeyframeTrack for the property they changed (resizeEntity
-// already does for width/height).
+// surfaces keep the track of the property they changed in step themselves —
+// in an app, through whatever owns the document, since a keyframe minted in
+// the world alone would never reach the project it was authored in.
import { store } from './store';
import {