Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add live mount functionality to the mounting process

konstantin-paulus091631e55986
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index e390b66f..55d3c0cc 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -68,6 +68,7 @@ export type NodeDeleteResult = { id: number };
export type MountRequest = {
code: string;
+ live?: boolean;
};
export type NodeInsertRequest = {
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index ab3a903e..f763e709 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -669,7 +669,7 @@ async function mediaWaveform(ref: string, opts: MediaPreviewOptions): Promise<vo
const MOUNT_EXTENSIONS = new Set([".tsx", ".jsx", ".ts", ".js"]);
-type MountOptions = { code?: string };
+type MountOptions = { code?: string; live?: boolean };
// Validates the (<path> | --code) pair shared by `mount` and `node insert`,
// then compiles the module. Compile errors fail here, before the app is contacted.
@@ -708,7 +708,7 @@ async function mountProject(path: string | undefined, opts: MountOptions): Promi
const stop = startSpinner("Mounting project");
try {
- await editor.mount.mutate({ code }, GENERATE);
+ await editor.mount.mutate({ code, live: opts.live }, GENERATE);
stop();
} catch (e) {
stop();
@@ -1070,10 +1070,14 @@ program
program
.command("mount")
.description(
- `Compile a Solid JSX project module and mount its roots into the canvas. Re-mounting reconciles rather than duplicates: each top-level element carries a \`key\`, and a root replaces the node with that key or creates it (a new <scene key> becomes the active scene and the camera focuses it). Long-running when the module declares AI assets (blocks until generation finishes). Compile errors fail before the app is contacted; on success, inspect the result with \`dapi context\` or \`dapi node tree\`.`,
+ `Compile a Solid JSX project module and mount its roots into the canvas. Re-mounting reconciles rather than duplicates: each top-level element carries a \`key\`, and a root replaces the node with that key or creates it (a new <scene key> becomes the active scene and the camera focuses it). By default the module's reactive graph is disposed once the mount lands (the entities stay, but signals, effects, and timers stop); pass --live to keep it running. Long-running when the module declares AI assets (blocks until generation finishes). Compile errors fail before the app is contacted; on success, inspect the result with \`dapi context\` or \`dapi node tree\`.`,
)
.argument("[path]", "path to a .tsx / .jsx / .ts / .js entry module")
.option("--code <str>", "inline module source; export default wrapper optional for bare JSX")
+ .option(
+ "--live",
+ "keep the reactive graph alive after mounting: signals, effects, and timers keep driving the mounted entities until the app closes or a later mount claims one of its root keys (same key -> the old run is unmounted)",
+ )
.action((path: string | undefined, opts: MountOptions) => mountProject(path, opts));
const asset = program
diff --git a/apps/web/src/components/engine/api/engine.ts b/apps/web/src/components/engine/api/engine.ts
index f83fe198..895936d7 100644
--- a/apps/web/src/components/engine/api/engine.ts
+++ b/apps/web/src/components/engine/api/engine.ts
@@ -104,6 +104,7 @@ export function createEngine(id: string) {
function dispose(): void {
stop();
+ world.liveMounts.disposeAll();
keyboard.dispose();
camera.dispose();
}
diff --git a/apps/web/src/components/engine/api/world.ts b/apps/web/src/components/engine/api/world.ts
index a0b99c89..b3f8ea38 100644
--- a/apps/web/src/components/engine/api/world.ts
+++ b/apps/web/src/components/engine/api/world.ts
@@ -8,6 +8,7 @@ import { createSignal } from "solid-js";
import { observeWorld } from "./observers";
import { createComponents } from "./components";
import { createHistory } from "../utils/history";
+import { createLiveMounts } from "../utils/live-mounts";
import { createStore } from "../db";
import { initAssets } from "./assets";
import { initFolders } from "./folders";
@@ -27,6 +28,7 @@ export function createEngineWorld(projectId: string, audioContext: AudioContext
projectId,
components: createComponents(),
history: createHistory(),
+ liveMounts: createLiveMounts(),
selection: {
scene: null as number | null,
tool: 0,
diff --git a/apps/web/src/components/engine/utils/index.ts b/apps/web/src/components/engine/utils/index.ts
index ca5006c4..4d4d9ab3 100644
--- a/apps/web/src/components/engine/utils/index.ts
+++ b/apps/web/src/components/engine/utils/index.ts
@@ -4,6 +4,7 @@
export * from './coordinates';
export * from './history';
+export * from './live-mounts';
export * from './time';
export * from './sort';
export * from './cursor';
diff --git a/apps/web/src/components/engine/utils/live-mounts.ts b/apps/web/src/components/engine/utils/live-mounts.ts
new file mode 100644
index 00000000..4c86635a
--- /dev/null
+++ b/apps/web/src/components/engine/utils/live-mounts.ts
@@ -0,0 +1,37 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Live JSX mounts (`dapi mount --live`) keep their Solid reactive graph
+ * running after the mount lands, so signals, effects, and timers keep
+ * driving the mounted entities. A run stays alive until the world is
+ * disposed or a later mount claims one of its root keys: the keyed root
+ * replacement already deletes its entities, so the superseded graph is
+ * disposed with them. Root keys are read live rather than snapshotted, so
+ * roots a running graph adds or removes later still reconcile.
+ */
+export type LiveMount = { rootKeys: () => string[]; dispose: () => void };
+
+export type LiveMounts = ReturnType<typeof createLiveMounts>;
+
+export function createLiveMounts() {
+ const mounts = new Set<LiveMount>();
+ return {
+ supersede(keys: string[]): void {
+ for (const mount of mounts) {
+ if (keys.some((key) => mount.rootKeys().includes(key))) {
+ mount.dispose();
+ mounts.delete(mount);
+ }
+ }
+ },
+ register(mount: LiveMount): void {
+ mounts.add(mount);
+ },
+ disposeAll(): void {
+ for (const mount of mounts) mount.dispose();
+ mounts.clear();
+ },
+ };
+}
diff --git a/apps/web/src/context/editor-api/mount.ts b/apps/web/src/context/editor-api/mount.ts
index 9666ca1e..3d35f3eb 100644
--- a/apps/web/src/context/editor-api/mount.ts
+++ b/apps/web/src/context/editor-api/mount.ts
@@ -14,15 +14,26 @@ import type { Accessor } from "solid-js";
import type { MountRequest, NodeInsertRequest } from "@diffusionstudio/cli/channels";
import type { Engine } from "@/components/engine";
+function documentRootKeys(world: Engine["world"], document: WorldDocument): string[] {
+ const keys: string[] = [];
+ for (const node of document.stage.children ?? []) {
+ if (node.kind !== "element") continue;
+ const key = world.components.Key[node.eid];
+ if (key !== undefined) keys.push(key);
+ }
+ return keys;
+}
+
/**
* `dapi mount` — evaluates a compiled project module and renders it directly into the ECS world
*/
export function handleMount(engine: Accessor<Engine>) {
- return async ({ code }: MountRequest): Promise<void> => {
+ return async ({ code, live }: MountRequest): Promise<void> => {
const e = engine();
const world = e.world;
const c = world.components;
let dispose = () => { };
+ let persisted = false;
try {
// Evaluate — top-level code (including top-level await) runs here.
@@ -41,6 +52,11 @@ export function handleMount(engine: Accessor<Engine>) {
throw error;
}
+ // The keyed root replacement above deleted any same-key entities, so
+ // graphs that were driving them are dead; dispose them before their
+ // next effect can write to a recycled entity id.
+ world.liveMounts.supersede(documentRootKeys(world, document));
+
try {
world.history.startTransaction("Commit document");
await document.commit();
@@ -48,12 +64,17 @@ export function handleMount(engine: Accessor<Engine>) {
world.history.commitTransaction();
}
+ if (live) {
+ world.liveMounts.register({ rootKeys: () => documentRootKeys(world, document), dispose });
+ persisted = true;
+ }
+
if (document.rootEid && hasComponent(world, document.rootEid, c.Scene)) {
switchActiveScene(world, document.rootEid);
e.camera.fitToActiveScene();
}
} finally {
- dispose?.();
+ if (!persisted) dispose?.();
}
};
}
diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.ts
index c4e3f5c5..dabd3ff5 100644
--- a/apps/web/src/utils/jsx.ts
+++ b/apps/web/src/utils/jsx.ts
@@ -278,6 +278,21 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
private queue: GenaiQueueItem[] = [];
+ // Set once `commit()` finishes. A live mount (`dapi mount --live`) keeps
+ // its reactive graph running past commit, so later writes flow through the
+ // same paths, untracked; otherwise the undo stack would fill with one
+ // entry per reactive update.
+ private committed = false;
+
+ private track<T>(fn: () => T): T {
+ if (!this.committed) return fn();
+ let result!: T;
+ this.world.history.untrack(() => {
+ result = fn();
+ });
+ return result;
+ }
+
public constructor(engine: Engine, target?: { parentEid: number }) {
this.stage = { kind: "root", eid: target?.parentEid };
this.engine = engine;
@@ -300,14 +315,9 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
{
const memo: GenerationMemo = new Map();
- const results = await Promise.allSettled(this.queue.map(async ({ node, ref, type }) => {
- if (node.kind !== "element" || hasComponent(world, node.eid, c.Deleted) || type !== "asset" || !ref) return;
- try {
- const asset = await resolveGeneratedAsset(world, ref, memo);
- await this.mountSource(node, asset.id);
- } finally {
- world.history.untrack(() => removeComponent(world, node.eid, c.Generating));
- }
+ const results = await Promise.allSettled(this.queue.map(({ node, ref, type }) => {
+ if (type !== "asset" || !ref) return;
+ return this.generateInto(node, ref, memo);
}));
assertAllSettled(results);
@@ -373,13 +383,36 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
setComponent(world, node.eid, c.Trim, trim);
}
}
+
+ this.queue.length = 0;
+ this.committed = true;
}
private get world(): EngineWorld {
return this.engine.world;
}
+ /**
+ * Resolves a generated asset and attaches it: during commit via the genai
+ * queue, or immediately when a live mount sets an asset ref after commit.
+ */
+ private async generateInto(node: DocumentNode, ref: AssetRef, memo: GenerationMemo): Promise<void> {
+ const world = this.world;
+ const c = world.components;
+ if (node.kind !== "element" || hasComponent(world, node.eid, c.Deleted)) return;
+ try {
+ const asset = await resolveGeneratedAsset(world, ref, memo);
+ await this.mountSource(node, asset.id);
+ } finally {
+ world.history.untrack(() => removeComponent(world, node.eid, c.Generating));
+ }
+ }
+
public createElement(tag: string): DocumentNode {
+ return this.track(() => this.createElementImpl(tag));
+ }
+
+ private createElementImpl(tag: string): DocumentNode {
const world = this.world;
const c = world.components;
@@ -441,6 +474,7 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
appendChild(world, fid, eid);
return node;
} case "captions": {
+ assert(!this.committed, "<captions> cannot be created after commit; re-mount instead");
const eid = createEntity(world);
const node: DocumentNode = { kind: "element", eid };
setComponent(world, eid, c.Geometry, GeometryType.TEXT);
@@ -533,6 +567,12 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
public replaceText(node: DocumentNode, text: string) {
assert(node.kind === "text", "replaceText target is not a text node");
node.text = text;
+ // Already-inserted text (a live mount updating a signal inside <text>)
+ // must land in the world too; insertNode only writes Chars on insert.
+ const parent = node.parent;
+ if (parent?.kind === "element") {
+ this.track(() => setComponent(this.world, parent.eid, this.world.components.Chars, text));
+ }
}
public isTextNode(node: DocumentNode): boolean {
@@ -541,7 +581,10 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
public setProperty(node: DocumentNode, name: string, value: unknown) {
if (name === "children" || name === "ref" || value === undefined || node.kind !== "element") return;
+ this.track(() => this.setPropertyImpl(node, name, value));
+ }
+ private setPropertyImpl(node: DocumentNode & { kind: "element" }, name: string, value: unknown) {
const world = this.world;
const c = world.components;
const eid = node.eid;
@@ -735,6 +778,7 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
} case "syncTo": {
assert(typeof value === "string", "`syncTo` must be a string" + `, value: ${value}`);
assert(value.trim().length > 0, "`syncTo` must be a non-empty string");
+ assert(!this.committed, "`syncTo` cannot change after commit; re-mount instead");
this.queue.push({ node, targetKey: value.trim(), type: "sync" });
break;
} case "transition": {
@@ -782,13 +826,16 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
if (!hasComponent(world, eid, c.Size)) {
resizeEntity(world, eid, placeholderSize(getAssetSpec(value)));
}
- this.queue.push({ node, ref: value, type: "asset" });
+ // Post-commit (live mount) nothing drains the queue; resolve now.
+ if (this.committed) this.generateInto(node, value, new Map()).catch(console.error);
+ else this.queue.push({ node, ref: value, type: "asset" });
break;
}
assert(typeof value === "string", "`src` must be a string" + `, value: ${value}`);
assert(value.trim().length > 0, "`src` must be a non-empty string");
- this.promises.push(this.mountSource(node, value));
+ if (this.committed) this.mountSource(node, value).catch(console.error);
+ else this.promises.push(this.mountSource(node, value));
break;
} case "objectFit":
assert(typeof value === "string", "`objectFit` must be a string" + `, value: ${value}`);
@@ -905,6 +952,10 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
}
public insertNode(parent: DocumentNode, node: DocumentNode) {
+ this.track(() => this.insertNodeImpl(parent, node));
+ }
+
+ private insertNodeImpl(parent: DocumentNode, node: DocumentNode) {
const c = this.world.components;
if (parent.kind === "root" && node.kind === "element") {
if (parent.eid === undefined) {
@@ -949,12 +1000,14 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
public removeNode(parent: DocumentNode, node: DocumentNode) {
const c = this.world.components;
- if (parent.kind === "element" && node.kind === "text") {
- removeComponent(this.world, parent.eid, c.Chars);
- } else if (parent.kind === "element" && node.kind === "element") {
- removeChild(this.world, node.eid, parent.eid);
- deleteEntity(this.world, node.eid);
- }
+ this.track(() => {
+ if (parent.kind === "element" && node.kind === "text") {
+ removeComponent(this.world, parent.eid, c.Chars);
+ } else if (parent.kind === "element" && node.kind === "element") {
+ removeChild(this.world, node.eid, parent.eid);
+ deleteEntity(this.world, node.eid);
+ }
+ });
parent.children = parent.children?.filter((child) => child !== node);
node.parent = undefined;
@@ -993,31 +1046,34 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
if (node.kind !== "element") return;
const asset = await resolveAsset(this.world, src);
- const world = this.world;
- const c = world.components;
- const eid = node.eid;
+ const transcript = asset.type === 'TRANSCRIPT' ? await resolveTranscript(asset) : null;
+
+ this.track(() => {
+ const world = this.world;
+ const c = world.components;
+ const eid = node.eid;
+
+ if (asset.type === 'TRANSCRIPT') {
+ // AssetId set here makes commit's caption phase skip transcription.
+ setComponent(world, eid, c.AssetId, asset.id);
+ setComponent(world, eid, c.Trim, transcriptTrim(transcript!));
+ } else if (asset.type === 'AUDIO') {
+ setComponent(world, eid, c.AssetId, asset.id);
+ resizeEntity(world, eid, { width: 500, height: 150 });
+ } else if (asset.type === 'IMAGE' || asset.type === 'VIDEO') {
+ const fid = createEntity(world);
+ setComponent(world, fid, c.Paint, asset.type === 'IMAGE' ? PaintType.IMAGE : PaintType.VIDEO);
+ setComponent(world, fid, c.ScaleMode, node.objectFit ? SCALE_MODE_MAP[node.objectFit] : ScaleMode.COVER);
+ setComponent(world, fid, c.AssetId, asset.id);
+ appendChild(world, fid, eid);
- if (asset.type === 'TRANSCRIPT') {
- const transcript = await resolveTranscript(asset);
- // AssetId set here makes commit's caption phase skip transcription.
- setComponent(world, eid, c.AssetId, asset.id);
- setComponent(world, eid, c.Trim, transcriptTrim(transcript));
- } else if (asset.type === 'AUDIO') {
- setComponent(world, eid, c.AssetId, asset.id);
- resizeEntity(world, eid, { width: 500, height: 150 });
- } else if (asset.type === 'IMAGE' || asset.type === 'VIDEO') {
- const fid = createEntity(world);
- setComponent(world, fid, c.Paint, asset.type === 'IMAGE' ? PaintType.IMAGE : PaintType.VIDEO);
- setComponent(world, fid, c.ScaleMode, node.objectFit ? SCALE_MODE_MAP[node.objectFit] : ScaleMode.COVER);
- setComponent(world, fid, c.AssetId, asset.id);
- appendChild(world, fid, eid);
-
- if (!hasComponent(world, eid, c.Size)) {
- resizeEntity(world, eid, { width: asset.width, height: asset.height });
+ if (!hasComponent(world, eid, c.Size)) {
+ resizeEntity(world, eid, { width: asset.width, height: asset.height });
+ }
}
- }
- setComponent(world, eid, c.Name, asset.name);
+ setComponent(world, eid, c.Name, asset.name);
+ });
}
}