Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor document structure to support text nodes and enhance editing capabilities

konstantin-paulus9b5bf09f3218
diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.ts
index f3963f8c..1ed99c88 100644
--- a/apps/web/src/engine/editor.ts
+++ b/apps/web/src/engine/editor.ts
@@ -14,11 +14,11 @@ import { getEntityChildren, getParentEntity, Source } from '@diffusionstudio/run
import { SOURCE_ATTR } from '@diffusionstudio/jsx';
import { createRoot } from 'solid-js';
-import { getRuntimeDocument, insert, withDocument } from '@diffusionstudio/reconciler';
+import { getRuntimeDocument, insert, isSceneNode, withDocument } from '@diffusionstudio/reconciler';
import type { PropValue } from '@diffusionstudio/jsx';
import type { Entity, World } from 'koota';
-import type { ProjectDocument, RuntimeDocument, SceneNode } from '@diffusionstudio/reconciler';
+import type { HostNode, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';
/**
* A property the editor changed, in the vocabulary of the JSX rather than of
@@ -217,7 +217,7 @@ export class DocumentEditor {
* every element created stamped pending and its tag and literal props
* noted in `created`. The document itself never learns it was watched.
*/
- private recording(created: Map<Entity, Recorded>): ProjectDocument<SceneNode> {
+ private recording(created: Map<Entity, Recorded>): ProjectDocument<HostNode> {
const document = this.document;
return {
stage: document.stage,
@@ -230,15 +230,15 @@ export class DocumentEditor {
return node;
},
setProperty: (node, name, value) => {
- const recorded = created.get(node.entity);
+ const recorded = isSceneNode(node) ? created.get(node.entity) : undefined;
if (recorded && name !== SOURCE_ATTR && name !== 'children' && name !== 'ref' && isPropValue(value)) {
recorded.props[name] = value;
}
document.setProperty(node, name, value);
},
- createTextNode: () => document.createTextNode(),
- replaceText: () => document.replaceText(),
- isTextNode: () => document.isTextNode(),
+ createTextNode: (text) => document.createTextNode(text),
+ replaceText: (node, text) => document.replaceText(node, text),
+ isTextNode: (node) => document.isTextNode(node),
insertNode: (parent, node, anchor) => document.insertNode(parent, node, anchor),
removeNode: (parent, node) => document.removeNode(parent, node),
getParentNode: (node) => document.getParentNode(node),
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 7523396f..a2382881 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -7,16 +7,20 @@ import {
ActiveScene,
appendChild,
Background,
+ Chars,
ClipsContent,
createEntity,
DEFAULT_BACKGROUND,
Color,
End,
+ FontStyle,
FrameRate,
Geometry,
GeometryType,
+ getEntityTree,
getParentEntity,
getParentNode,
+ isText,
ItemIndex,
Name,
parseColor,
@@ -35,6 +39,9 @@ import {
setCameraMatrix,
Start,
switchActiveScene,
+ TextAlign,
+ TextBaseline,
+ TextStyle,
} from '@diffusionstudio/runtime';
import { parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
@@ -46,6 +53,17 @@ export interface SceneNode {
readonly entity: Entity;
}
+export interface TextNode {
+ text: string;
+ parent: SceneNode | null;
+}
+
+export type HostNode = SceneNode | TextNode;
+
+export function isSceneNode(node: HostNode): node is SceneNode {
+ return 'entity' in node;
+}
+
const TIME_TRAITS = {
start: Start,
end: End,
@@ -53,6 +71,29 @@ const TIME_TRAITS = {
sourceOut: SourceOut,
} as const;
+const FONT_STYLES: Record<string, FontStyle> = {
+ normal: FontStyle.NORMAL,
+ italic: FontStyle.ITALIC,
+ oblique: FontStyle.OBLIQUE,
+};
+
+const TEXT_ALIGNS: Record<string, TextAlign> = {
+ left: TextAlign.LEFT,
+ center: TextAlign.CENTER,
+ right: TextAlign.RIGHT,
+};
+
+const TEXT_BASELINES: Record<string, TextBaseline> = {
+ top: TextBaseline.TOP,
+ middle: TextBaseline.MIDDLE,
+ bottom: TextBaseline.BOTTOM,
+};
+
+const FONT_WEIGHTS: Record<string, string> = {
+ normal: '400',
+ bold: '700',
+};
+
function toNumber(value: unknown) {
if (value === undefined || value === null) {
return undefined;
@@ -70,10 +111,10 @@ function toSeconds(value: unknown): number | undefined {
return parseTime(value);
}
-export class RuntimeDocument implements ProjectDocument<SceneNode> {
- /** The mount root. Both it and the <stage> element stand for the Stage entity. */
+export class RuntimeDocument implements ProjectDocument<HostNode> {
public readonly stage: SceneNode;
private readonly world: World;
+ private readonly texts = new Map<Entity, TextNode[]>();
public constructor(world: World) {
this.world = world;
@@ -112,24 +153,46 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
resizeEntity(this.world, entity, { width: 100, height: 100 });
break;
}
+ case 'text': {
+ // No Size: a text without one sizes itself to its glyphs.
+ entity = createEntity(this.world);
+ entity.add(Geometry);
+ entity.set(Geometry, { value: GeometryType.TEXT });
+ entity.add(Position);
+ entity.set(Position, { x: 0, y: 0 });
+ entity.add(Chars);
+ entity.add(TextStyle);
+ break;
+ }
default:
- throw new Error(`<${tag}> is not supported yet (only <stage>, <scene> and <rect>).`);
+ throw new Error(`<${tag}> is not supported yet (only <stage>, <scene>, <rect> and <text>).`);
}
return { entity };
}
- public createTextNode(): SceneNode {
- throw new Error('Text children are not supported.');
+ public createTextNode(text: string): TextNode {
+ return { text, parent: null };
}
- public replaceText(): void { }
+ public replaceText(node: HostNode, text: string): void {
+ if (isSceneNode(node)) return;
+ node.text = text;
+
+ if (node.parent) {
+ node.parent.entity.add(Chars);
+ node.parent.entity.set(Chars, { value: this.buildText(node) });
+ }
+ }
- public isTextNode(): boolean {
- return false;
+ public isTextNode(node: HostNode): boolean {
+ return !isSceneNode(node);
}
- public setProperty({ entity }: SceneNode, name: string, value: unknown): void {
+ public setProperty(node: HostNode, name: string, value: unknown): void {
+ if (!isSceneNode(node)) return;
+ const { entity } = node;
+
switch (name) {
case SOURCE_ATTR: {
if (typeof value !== 'string' || !value) {
@@ -181,7 +244,8 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
entity.set(trait, { value: secondsToFrames(seconds, fps) });
return;
}
- case 'fill': {
+ case 'fill':
+ case 'color': {
const color = parseColor(value);
if (color === null) {
@@ -193,6 +257,41 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
entity.set(Color, { value: color });
return;
}
+ case 'fontSize': {
+ const size = toNumber(value);
+ entity.add(TextStyle);
+ entity.set(TextStyle, { fontSize: size !== undefined && size > 0 ? Math.round(size) : undefined });
+ return;
+ }
+ case 'fontFamily': {
+ const family = typeof value === 'string' ? value.trim() : '';
+ entity.add(TextStyle);
+ entity.set(TextStyle, { fontFamily: family || undefined });
+ return;
+ }
+ case 'fontWeight': {
+ // Authored as a CSS keyword or a number; the trait keeps the numeric string.
+ const weight = value === undefined || value === null ? '' : String(value).trim();
+ const numeric = FONT_WEIGHTS[weight] ?? weight;
+ entity.add(TextStyle);
+ entity.set(TextStyle, { fontWeight: numeric && Number.isFinite(Number(numeric)) ? numeric : undefined });
+ return;
+ }
+ case 'fontStyle': {
+ entity.add(TextStyle);
+ entity.set(TextStyle, { fontStyle: typeof value === 'string' ? FONT_STYLES[value] : undefined });
+ return;
+ }
+ case 'textAlign': {
+ entity.add(TextStyle);
+ entity.set(TextStyle, { textAlign: typeof value === 'string' ? TEXT_ALIGNS[value] : undefined });
+ return;
+ }
+ case 'textBaseline': {
+ entity.add(TextStyle);
+ entity.set(TextStyle, { textBaseline: typeof value === 'string' ? TEXT_BASELINES[value] : undefined });
+ return;
+ }
case 'background': {
const color = parseColor(value);
this.stage.entity.set(Background, { value: color ?? DEFAULT_BACKGROUND });
@@ -215,12 +314,26 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
}
/**
- * Node handles are minted per lookup, so every comparison here is between
- * entities: two handles for the same node are never the same object.
+ * Entity handles are minted per lookup, so every comparison here is between
+ * entities: two handles for the same node are never the same object. Text
+ * nodes are the exception, held by identity (see `TextNode`).
*/
- public insertNode(parent: SceneNode, node: SceneNode, anchor?: SceneNode): void {
+ public insertNode(parent: HostNode, node: HostNode, anchor?: HostNode): void {
+ if (!isSceneNode(parent)) {
+ throw new Error('Text cannot contain children.');
+ }
+
+ if (!isSceneNode(node)) {
+ this.insertText(parent, node, anchor);
+ return;
+ }
+
if (parent.entity === node.entity) return;
+ if (isText(parent.entity)) {
+ throw new Error('<text> only takes text children.');
+ }
+
// Scenes own a timeline, so one inside another has no coherent reading.
if (node.entity.has(Scene) && !parent.entity.has(Stage)) {
throw new Error('<scene> is only allowed as a direct child of <stage>.');
@@ -238,8 +351,8 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
appendChild(this.world, node.entity, parent.entity);
}
- const siblings = this.childEntities(parent).filter((sibling) => sibling !== node.entity);
- const at = anchor ? siblings.indexOf(anchor.entity) : -1;
+ const siblings = getEntityChildren(this.world, parent.entity).filter((sibling) => sibling !== node.entity);
+ const at = anchor && isSceneNode(anchor) ? siblings.indexOf(anchor.entity) : -1;
if (at === -1) siblings.push(node.entity);
else siblings.splice(at, 0, node.entity);
for (const [index, sibling] of siblings.entries()) {
@@ -248,7 +361,12 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
}
}
- public removeNode(_parent: SceneNode, node: SceneNode): void {
+ public removeNode(_parent: HostNode, node: HostNode): void {
+ if (!isSceneNode(node)) {
+ this.removeText(node);
+ return;
+ }
+
if (node.entity.has(Stage)) {
for (const child of this.children(node)) {
this.removeNode(node, child);
@@ -261,23 +379,41 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
if (this.world.get(ActiveScene)?.entity === node.entity) {
switchActiveScene(this.world, null);
}
+ // Destroy cascades through the subtree; the text nodes held for it go too.
+ for (const entity of getEntityTree(this.world, node.entity)) {
+ const texts = this.texts.get(entity);
+ if (texts === undefined) continue;
+ for (const text of texts) {
+ text.parent = null;
+ }
+ this.texts.delete(entity);
+ }
// ChildOf auto-destroys orphans, so paints go with it.
node.entity.destroy();
}
- public getParentNode(node: SceneNode): SceneNode | undefined {
+ public getParentNode(node: HostNode): SceneNode | undefined {
+ if (!isSceneNode(node)) return node.parent ?? undefined;
const parent = getParentEntity(node.entity);
return parent === null ? undefined : { entity: parent };
}
- public getFirstChild(node: SceneNode): SceneNode | undefined {
- return this.children(node)[0];
+ public getFirstChild(node: HostNode): HostNode | undefined {
+ if (!isSceneNode(node)) return undefined;
+ return this.texts.get(node.entity)?.[0] ?? this.children(node)[0];
}
- public getNextSibling(node: SceneNode): SceneNode | undefined {
+ public getNextSibling(node: HostNode): HostNode | undefined {
+ if (!isSceneNode(node)) {
+ // A text entity has only text nodes to iterate.
+ const texts = node.parent ? this.texts.get(node.parent.entity) : undefined;
+ const at = texts?.indexOf(node) ?? -1;
+ return at === -1 ? undefined : texts![at + 1];
+ }
+
const parent = this.getParentNode(node);
if (parent === undefined) return undefined;
- const siblings = this.childEntities(parent);
+ const siblings = getEntityChildren(this.world, parent.entity);
const at = siblings.indexOf(node.entity);
if (at === -1) return undefined;
const next = siblings[at + 1];
@@ -290,12 +426,54 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
/** Children of `parent` in draw order, as handles. */
private children(parent: SceneNode): SceneNode[] {
- return this.childEntities(parent).map((entity) => ({ entity }));
+ return getEntityChildren(this.world, parent.entity).map((entity) => ({ entity }));
}
- /** Children of `parent` in draw order. */
- private childEntities(parent: SceneNode): Entity[] {
- return getEntityChildren(this.world, parent.entity);
+ private insertText(parent: SceneNode, node: TextNode, anchor?: HostNode): void {
+ if (!isText(parent.entity)) {
+ throw new Error('Only <text> takes text children.');
+ }
+
+ if (node.parent && node.parent.entity !== parent.entity) {
+ this.removeText(node);
+ }
+
+ let texts = this.texts.get(parent.entity);
+ if (texts === undefined) {
+ texts = [];
+ this.texts.set(parent.entity, texts);
+ }
+
+ const current = texts.indexOf(node);
+ if (current !== -1) texts.splice(current, 1);
+ const at = anchor && !isSceneNode(anchor) ? texts.indexOf(anchor) : -1;
+ if (at === -1) texts.push(node);
+ else texts.splice(at, 0, node);
+
+ node.parent = parent;
+ node.parent.entity.add(Chars);
+ node.parent.entity.set(Chars, { value: this.buildText(node) });
+ }
+
+ private removeText(node: TextNode): void {
+ const parent = node.parent;
+ if (parent === null) return;
+
+ const texts = this.texts.get(parent.entity);
+ const at = texts?.indexOf(node) ?? -1;
+ if (at !== -1) texts!.splice(at, 1);
+ node.parent = null;
+
+ if (parent.entity.isAlive()) {
+ parent.entity.add(Chars);
+ parent.entity.set(Chars, { value: this.buildText(node) });
+ }
+ }
+
+ private buildText(node: TextNode): string {
+ if (isSceneNode(node) || node.parent === null) return '';
+
+ return (this.texts.get(node.parent.entity) ?? []).map((node) => node.text).join('');
}
}
diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.md
index e1b22337..989c7f87 100644
--- a/packages/runtime/MIGRATION.md
+++ b/packages/runtime/MIGRATION.md
@@ -131,7 +131,7 @@ apps/web now depends on `@diffusionstudio/runtime` and `@diffusionstudio/koota-s
**The `key` prop went with it**, for the same reason: it was a within-render label letting elements point at each other (`syncTo` named a clip's `key`), which is the job `id` already does, and unlike `key` an id survives an edit that renumbers the file. So `syncTo` now names an `id`. Dropped from `PatchProps`, `CommonProps`, and `PATCH_PROP_KEYS`. The runtime's `Key` trait is untouched and unrelated: it is the ECS-side stable identity that clone/paste external refs key off (entity ids do not survive delete/undo), and nothing on the JSX path ever wrote it.
- **Element insertion (write-back of new elements).** Editing commands live app-side in `DocumentEditor` (`apps/web/src/engine/editor.ts`, one per world via `getDocumentEditor(world)`, hook `useEditor()`; the reconciler exports `insert`/`withDocument` for it), not in `RuntimeDocument`, which stays a plain `ProjectDocument` host: `onEdit`, `editProperty`, `reportEdit`, `insertElement`, `restamp`, `discardPending`. `editor.insertElement(parent, () => <Rect x={10} width={200} height={300} />, anchor?)` renders the thunk through Solid's universal `insert` under `withDocument` with a recording proxy of the document (same host, every `createElement` stamped pending and `setProperty` noted; the document never learns it was watched), so it goes through the document's own `createElement`/`setProperty`/`insertNode` and is a first-class node on the canvas at once, no remount. The app's JSX compiles for the DOM, so lowercase `<rect>` is unusable there; the PascalCase components (`Rect`, `Scene`, ... now exported from the reconciler index) compile identically in both modes and are the spelling to use. `createElement` lowercases the first letter, so both spellings reach the same recipe. While the thunk renders, every element created is stamped with a pending source (`pending#n`, colon-free so `parseSource` rejects it as an address) and its tag + literal props recorded; afterwards one `InsertEdit` per element is reported in creation order (`EntityEdit` is now `PropEdit | InsertEdit`, discriminated by `kind`), a nested `<Scene><Rect/></Scene>` naming its parent by pending source, and `before` set to the next pre-existing sibling. Returns the top-level entities created. The desktop `applyEdits` grew a matching `SourceInsert` (`SourceEdit = SourceSet | SourceInsert`): it appends `<tag {...props} id="…" />` under the parent (or in front of `before`, which must be a direct child) as text edits at positions ts-morph supplies (`getIndentationText`/`getChildIndentationText`/`isFirstNodeOnLine`, with `manipulationSettings.indentationText` matched to the file first since ts-morph does not detect it; it has no insert-child or open-self-closing operation short of re-printing the body), props via the same `setProp`/`addAttribute` path as prop edits, allocates the id on the spot, and answers with `ids[pending] = real`. A pending parent earlier in the same batch resolves through that map, so a rect drawn into a just-drawn scene lands in one write. The `EditWriter` class (`apps/web/src/projects/edits.ts`) sends inserts first, folds prop edits into a not-yet-flushed insert, holds back edits (and child inserts) addressed through an in-flight pending name until the answer arrives, then re-keys them and re-stamps entities via `DocumentEditor.restamp`; an insert the file would not take (`skipped`) is undone on the canvas via `DocumentEditor.discardPending`. Not built yet: `<text>` in `createElement` (Text tool), and the DrawOverlay port itself.
+ **Element insertion (write-back of new elements).** Editing commands live app-side in `DocumentEditor` (`apps/web/src/engine/editor.ts`, one per world via `getDocumentEditor(world)`, hook `useEditor()`; the reconciler exports `insert`/`withDocument` for it), not in `RuntimeDocument`, which stays a plain `ProjectDocument` host: `onEdit`, `editProperty`, `reportEdit`, `insertElement`, `restamp`, `discardPending`. `editor.insertElement(parent, () => <Rect x={10} width={200} height={300} />, anchor?)` renders the thunk through Solid's universal `insert` under `withDocument` with a recording proxy of the document (same host, every `createElement` stamped pending and `setProperty` noted; the document never learns it was watched), so it goes through the document's own `createElement`/`setProperty`/`insertNode` and is a first-class node on the canvas at once, no remount. The app's JSX compiles for the DOM, so lowercase `<rect>` is unusable there; the PascalCase components (`Rect`, `Scene`, ... now exported from the reconciler index) compile identically in both modes and are the spelling to use. `createElement` lowercases the first letter, so both spellings reach the same recipe. While the thunk renders, every element created is stamped with a pending source (`pending#n`, colon-free so `parseSource` rejects it as an address) and its tag + literal props recorded; afterwards one `InsertEdit` per element is reported in creation order (`EntityEdit` is now `PropEdit | InsertEdit`, discriminated by `kind`), a nested `<Scene><Rect/></Scene>` naming its parent by pending source, and `before` set to the next pre-existing sibling. Returns the top-level entities created. The desktop `applyEdits` grew a matching `SourceInsert` (`SourceEdit = SourceSet | SourceInsert`): it appends `<tag {...props} id="…" />` under the parent (or in front of `before`, which must be a direct child) as text edits at positions ts-morph supplies (`getIndentationText`/`getChildIndentationText`/`isFirstNodeOnLine`, with `manipulationSettings.indentationText` matched to the file first since ts-morph does not detect it; it has no insert-child or open-self-closing operation short of re-printing the body), props via the same `setProp`/`addAttribute` path as prop edits, allocates the id on the spot, and answers with `ids[pending] = real`. A pending parent earlier in the same batch resolves through that map, so a rect drawn into a just-drawn scene lands in one write. The `EditWriter` class (`apps/web/src/projects/edits.ts`) sends inserts first, folds prop edits into a not-yet-flushed insert, holds back edits (and child inserts) addressed through an in-flight pending name until the answer arrives, then re-keys them and re-stamps entities via `DocumentEditor.restamp`; an insert the file would not take (`skipped`) is undone on the canvas via `DocumentEditor.discardPending`. `<text>` is hosted too: the recipe is TEXT geometry + `Chars` + `TextStyle` and no `Size` (a text without one sizes itself to its glyphs, which is what a click-inserted text wants); `color` writes the entity's `Color` (the intrinsic fill the text renderer already draws glyphs with, so no paint child), and `fontSize`/`fontFamily`/`fontWeight`/`fontStyle`/`textAlign`/`textBaseline` write `TextStyle` (an unset or invalid value clears the field to `undefined`, i.e. the renderer's default). Text content travels as Solid text nodes, and the runtime has no node for those, so a plain `TextNode` object (`{ text, parent }`, `HostNode = SceneNode | TextNode`, `isSceneNode` narrows) stands in for the DOM text node the way the original bitecs host's `{ kind: "text" }` did: held by identity (Solid compares handles with `===` and calls `replaceText(getFirstChild(parent))`), kept per text entity in an ordered list, and the list's join is the entity's `Chars`, so `<Text>Hello {name()}</Text>` reconciles through `replaceText`/`insertNode`/`removeNode` on the text nodes alone. Text under anything but a text entity throws, as does an entity under a text. `insertElement`'s recording proxy passes text nodes through untouched (only `SceneNode`s are stamped/recorded). Not built yet: the DrawOverlay port itself.
**Left stale on purpose** (deferred to a docs/examples pass): all eight `examples/*.tsx` still open with `<rect scene="...">` and no `<stage>`, so `tsc -p examples --noEmit` fails on them; `reference/jsx/{scene,roots}.md` still describes the pre-`<stage>` model where a mount grafts roots into an existing document; and `reference/jsx/{audio-sync,video,audio,elements}.md` still document `key` as the `syncTo` target. Deliberately not half-migrated: `reference/jsx` is stale as a whole, and patching the `key` rows alone would hide that.