The VS Code of video editing, built for agents.
TypeScript
Enhance JSX composition handling and improve DOM integration
konstantin-paulus2272d029ae59
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.tsindex c86ee4d1..0ec3820e 100644--- a/apps/desktop/src/projects.ts+++ b/apps/desktop/src/projects.ts@@ -27,7 +27,7 @@ import type { BuildOptions, Plugin } from "esbuild";import { mainBridge } from "./main-manager";import { MAIN_CHANNELS } from "./main-channels";import { applyEdits, stampProject } from "./edit";-import { sourcePlugin } from "./source";+import { canonicalizeTagsPlugin, sourcePlugin } from "./source";import type { CompileResult, FsEntry, FsStat, ProjectInfo, SourceEdit, WriteResult } from "./main-channels";import type { SourceContext } from "./edit";@@ -615,7 +615,7 @@ const babelOptions = (file: string, filename: string): TransformOptions => ({configFile: false,// Runs before the presets, which is what it needs: Solid's transform// replaces the JSX trees this stamps.- plugins: [[sourcePlugin, { file }]],+ plugins: [[sourcePlugin, { file }], canonicalizeTagsPlugin],presets: [[preset("babel-preset-solid"), { generate: "universal", moduleName: RUNTIME_MODULE }],[preset("@babel/preset-typescript"), { onlyRemoveTypeImports: true }],diff --git a/apps/desktop/src/source.ts b/apps/desktop/src/source.tsindex 991b6e0b..91caf47f 100644--- a/apps/desktop/src/source.ts+++ b/apps/desktop/src/source.ts@@ -11,13 +11,113 @@// disagreed, a drag would write to the wrong element; that invariant is why// these two modules are read together.-import { ID_ATTR, LOOP_ATTR, SOURCE_ATTR, formatSource, isCompositionTag, isLoopTag } from "@diffusionstudio/jsx";+import { COMPOSITION_TAGS, ID_ATTR, LOOP_ATTR, SOURCE_ATTR, formatSource, isCompositionTag, isLoopTag } from "@diffusionstudio/jsx";-import type { PluginObj, types as t } from "@babel/core";+import type { NodePath, PluginObj, types as t } from "@babel/core";/** What babel hands a plugin factory. */type BabelApi = { types: typeof import("@babel/core").types };+// Composition tags are authored in camelCase but must reach the renderer as+// PascalCase components. Lowercase is reserved at runtime for real DOM nodes+// inside <html>/<htmlPaint>.+const COMPOSITION_COMPONENTS = new Map<string, string>(+ COMPOSITION_TAGS.map((tag) => [tag, tag.charAt(0).toUpperCase() + tag.slice(1)]),+);++// These composition names also exist in SVG. An SVG fragment is DOM content+// only while it has an SVG container in the same JSX tree; elsewhere the tag+// keeps its composition meaning.+const AMBIGUOUS_SVG_TAGS: ReadonlySet<string> = new Set(["rect", "text", "image"]);+const SVG_CONTAINERS: ReadonlySet<string> = new Set([+ "svg", "g", "defs", "symbol", "marker", "mask", "clipPath", "pattern",+ "filter", "linearGradient", "radialGradient", "textPath", "tspan", "switch",+]);++const PASCAL_ELEMENTS = new Map(+ [...COMPOSITION_COMPONENTS].map(([camel, pascal]) => [pascal, camel]),+);++const SOLID_CONTROL_FLOW: ReadonlySet<string> = new Set([+ "For", "Show", "Switch", "Match", "Suspense", "SuspenseList", "Index", "ErrorBoundary",+]);++function hasSvgAncestor(path: NodePath<t.JSXElement>, types: typeof import("@babel/core").types): boolean {+ return path.findParent(+ (parent) => parent.isJSXElement()+ && types.isJSXIdentifier(parent.node.openingElement.name)+ && SVG_CONTAINERS.has(parent.node.openingElement.name.name),+ ) !== null;+}++function isSvgCollision(path: NodePath<t.JSXElement>, name: string, types: typeof import("@babel/core").types): boolean {+ return AMBIGUOUS_SVG_TAGS.has(name) && hasSvgAncestor(path, types);+}++/**+ * Rewrites authored composition intrinsics to aliased PascalCase imports+ * before babel-preset-solid consumes the JSX tree. Aliases prevent a user+ * binding named e.g. `Rect` from capturing an authored `<rect>`.+ */+export function canonicalizeTagsPlugin({ types }: BabelApi): PluginObj {+ return {+ name: "jsx-canonical-composition-tags",+ visitor: {+ // Babel merges plugin and preset visitors. Rewriting the complete tree+ // on Program enter ensures Solid cannot consume a parent before this+ // pass reaches its descendants.+ Program(program) {+ const aliases = new Map<string, string>();++ program.traverse({+ JSXElement(path) {+ const name = path.node.openingElement.name;+ if (!types.isJSXIdentifier(name)) return;++ if (/^[A-Z]/.test(name.name) && !path.scope.hasBinding(name.name)) {+ const camel = PASCAL_ELEMENTS.get(name.name);+ if (camel !== undefined) {+ throw path.buildCodeFrameError(+ `<${name.name}> is not a tag; write the composition element as <${camel}>`,+ );+ }+ if (SOLID_CONTROL_FLOW.has(name.name)) {+ throw path.buildCodeFrameError(+ `<${name.name}> needs an import: add \`import { ${name.name} } from "solid-js"\``,+ );+ }+ return;+ }++ const component = COMPOSITION_COMPONENTS.get(name.name);+ if (component === undefined || isSvgCollision(path, name.name, types)) return;++ let alias = aliases.get(component);+ if (alias === undefined) {+ alias = program.scope.generateUidIdentifier(component).name;+ aliases.set(component, alias);+ }++ path.node.openingElement.name = types.jsxIdentifier(alias);+ if (path.node.closingElement) {+ path.node.closingElement.name = types.jsxIdentifier(alias);+ }+ },+ });++ if (aliases.size === 0) return;+ const specifiers = [...aliases].map(([name, alias]) =>+ types.importSpecifier(types.identifier(alias), types.identifier(name)),+ );+ program.unshiftContainer(+ "body",+ types.importDeclaration(specifiers, types.stringLiteral("@diffusionstudio/jsx")),+ );+ },+ },+ };+}+const jsxTagName = (element: t.JSXElement): string | undefined => {const name = element.openingElement.name;return name.type === "JSXIdentifier" ? name.name : undefined;@@ -51,7 +151,11 @@ export function sourcePlugin({ types }: BabelApi, { file }: { file: string }): Pconst opening = path.node.openingElement;const name = opening.name;- if (name.type !== "JSXIdentifier" || !isCompositionTag(name.name)) return;+ if (+ name.type !== "JSXIdentifier"+ || !isCompositionTag(name.name)+ || isSvgCollision(path, name.name, types)+ ) return;const loop = path.findParent((parent) => parent.isJSXElement() && isLoopTag(jsxTagName(parent.node) ?? ""),diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.tsindex 1548f5d5..ebebcfff 100644--- a/apps/web/src/engine/editor.ts+++ b/apps/web/src/engine/editor.ts@@ -16,9 +16,10 @@ import { createRoot } from 'solid-js';import { authoredElement, authoredTree, getRuntimeDocument, insert, isSceneNode, renderAuthored, withDocument } from '@diffusionstudio/reconciler';+import type { HostNode } from '@diffusionstudio/runtime';import type { PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';import type { Entity, World } from 'koota';-import type { AuthoredTree, HostNode, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';+import type { AuthoredTree, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';/*** A value an edit can carry to the file: what a source spells as a literal,@@ -794,7 +795,7 @@ export class DocumentEditor {stage: document.stage,createElement: (tag) => {const node = document.createElement(tag);- if (node !== document.stage) {+ if (isSceneNode(node) && node !== document.stage) {document.setProperty(node, SOURCE_ATTR, nextPendingSource());created.set(node.entity, { tag: tag.charAt(0).toLowerCase() + tag.slice(1), props: {} });}diff --git a/packages/encoder/package.json b/packages/encoder/package.jsonindex 319639ac..e8b797fa 100644--- a/packages/encoder/package.json+++ b/packages/encoder/package.json@@ -1,7 +1,7 @@{"name": "@diffusionstudio/encoder","version": "0.1.0",- "description": "Offline video/audio/image encoding over @diffusionstudio/runtime worlds via mediabunny. Needs a browser-grade context (OffscreenCanvas, OfflineAudioContext, AudioWorklet, SharedArrayBuffer).",+ "description": "Offline video/audio/image encoding over @diffusionstudio/runtime worlds via mediabunny. Needs a browser-grade context (HTMLCanvasElement, OfflineAudioContext, AudioWorklet, SharedArrayBuffer).","license": "MPL-2.0","type": "module","sideEffects": false,diff --git a/packages/encoder/src/encoder.ts b/packages/encoder/src/encoder.tsindex ef18a665..b1a94294 100644--- a/packages/encoder/src/encoder.ts+++ b/packages/encoder/src/encoder.ts@@ -79,8 +79,12 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {const sceneSubtree = getEntityTree(sourceWorld, sourceScene);const subtreeRecords = sceneSubtree.map(entity => serializeEntity(entity));- // Offscreen canvas- const offscreenCanvas = new OffscreenCanvas(width, height);+ // HTML paints need their layout roots attached to the destination canvas.+ const offscreenCanvas = document.createElement('canvas');+ offscreenCanvas.width = width;+ offscreenCanvas.height = height;+ offscreenCanvas.style.cssText = 'position:fixed;left:-100000px;top:0;pointer-events:none;';+ document.body.appendChild(offscreenCanvas);const offscreenCtx = offscreenCanvas.getContext('2d')!;// Offline audio context@@ -342,9 +346,9 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {};} finally {URL.revokeObjectURL(audioWorkletUrl);- // Free the graphs + hosts realized above (this offline world is- // discarded, so nothing else disposes them — an HtmlHost otherwise- // leaks a canvas on document.body).+ offscreenCanvas.remove();+ // Free the graphs and DOM roots realized above before discarding the+ // offline world.mounts?.disposeAll();// Release decoder file handles/caches, then the world slot itself// (koota caps live worlds at 16, so throwaway offline worlds mustdiff --git a/packages/encoder/src/image-encoder.ts b/packages/encoder/src/image-encoder.tsindex e42ce535..fb20262d 100644--- a/packages/encoder/src/image-encoder.ts+++ b/packages/encoder/src/image-encoder.ts@@ -55,7 +55,11 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncoderootRecord.End -= rootStart;}- const offscreenCanvas = new OffscreenCanvas(2, 2);+ const offscreenCanvas = document.createElement('canvas');+ offscreenCanvas.width = 2;+ offscreenCanvas.height = 2;+ offscreenCanvas.style.cssText = 'position:fixed;left:-100000px;top:0;pointer-events:none;';+ document.body.appendChild(offscreenCanvas);const offscreenCtx = offscreenCanvas.getContext('2d')!;// Never started — image capture is video-only; the context just satisfies@@ -218,9 +222,9 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncodeerror: e instanceof Error ? e : new Error('Unknown error'),};} finally {- // Free the graphs + hosts realized above (this offline world is- // discarded; realized HtmlHosts otherwise leak a canvas on- // document.body), then release decoders and the world slot itself+ offscreenCanvas.remove();+ // Free the graphs and DOM roots realized above, then release decoders+ // and the world slot itself// (koota caps live worlds at 16).mounts?.disposeAll();disposeDecoders(world, world.get(Root)!);@@ -236,8 +240,10 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncode};}-async function toBase64Png(canvas: OffscreenCanvas): Promise<string> {- const blob = await canvas.convertToBlob({ type: 'image/png' });+async function toBase64Png(canvas: HTMLCanvasElement): Promise<string> {+ const blob = await new Promise<Blob>((resolve, reject) => {+ canvas.toBlob((value) => value ? resolve(value) : reject(new Error('Could not encode PNG')), 'image/png');+ });const dataUrl = await new Promise<string>((resolve, reject) => {const reader = new FileReader();reader.onload = () => resolve(reader.result as string);diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 095e8a56..b68810a5 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -643,7 +643,7 @@ export type HtmlPaintProps = PaintProps & {children?: SolidJSX.Element;};-/** `<Html>` — a rectangle carrying an `<HtmlPaint>` with the given children. */+/** `<Html>` — a rectangle whose intrinsic paint draws the given DOM children. */export type HtmlProps = CommonProps & Pick<HtmlPaintProps, "children">;// HTMLCanvasElement without requiring the DOM lib (this package alsodiff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 2b112c69..8b8646c7 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,28 +3,15 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';+import { SVGElements } from 'solid-js/web';-import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';+import type { CameraMatrix, PropertyPath, SceneNode } from '@diffusionstudio/runtime';import type { AnimatableProperty, AssetRef } from '@diffusionstudio/jsx';-import { trait, type Entity, type World } from 'koota';+import type { Entity, World } from 'koota';import type { ProjectDocument } from './host';-/**- * One element of the document, and its place in it. `parent` and `children`- * are the tree the reconciler reads back: text nodes and element nodes in the- * one order they were inserted in, which is what the document answers- * `getFirstChild` and `getNextSibling` from.- */-export interface SceneNode {- readonly entity: Entity;- tag: string;- props: Record<string, unknown>;- parent: SceneNode | null;- children: HostNode[];-}-/*** The node a `<surface>` or `<surfacePaint>` element creates: what its `ref`* receives, like any other element's ref receives its node. `canvas` is the@@ -36,15 +23,6 @@ export interface SurfaceNode extends SceneNode {readonly canvas: HTMLCanvasElement | null;}-export interface TextNode {- text: string;- parent: SceneNode | null;-}--export type HostNode = SceneNode | TextNode;--const Host = trait(() => null as SceneNode | null);-/** Props that address or wire an element rather than describe it. */const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);@@ -71,7 +49,7 @@ export interface AuthoredElement {*/export function authoredElement(entity: Entity): AuthoredElement | undefined {const node = entity.get(Host);- if (!node || node.tag === '') return undefined;+ if (!node || !node.native || node.tag === '') return undefined;const text = isText(entity) ? entity.get(Chars)?.value : undefined;return { tag: node.tag, props: { ...node.props }, ...(text ? { text } : {}) };@@ -101,15 +79,19 @@ export function authoredTree(world: World, entity: Entity): AuthoredTree | undefreturn { ...element, children };}-export function isSceneNode(node: HostNode): node is SceneNode {- return 'entity' in node;+export function isSceneNode(node: SceneNode | undefined): node is SceneNode {+ return typeof node === 'object' && node !== null && 'entity' in node;+}++function isDomText(node: SceneNode | undefined): node is SceneNode & { readonly domNode: Text } {+ return typeof Text !== 'undefined' && node?.domNode instanceof Text;}/*** The text nodes among `node`'s children, in order; empty when it holds none.*/-function textParts(node: SceneNode): TextNode[] {- return node.children.filter((child): child is TextNode => !isSceneNode(child));+function textParts(node: SceneNode): (SceneNode & { readonly domNode: Text })[] {+ return node.children.filter(isDomText);}/**@@ -121,14 +103,14 @@ function syncChars(node: SceneNode): void {const { entity } = node;if (!entity.isAlive() || !isText(entity)) return;entity.add(Chars);- entity.set(Chars, { value: textParts(node).map((part) => part.text).join('') });+ entity.set(Chars, { value: textParts(node).map((part) => part.domNode.data).join('') });}/*** Takes `node` out of the children of whatever parent it has, and refreshes* that parent's text.*/-function detach(node: HostNode): void {+function detach(node: SceneNode): void {const parent = node.parent;if (parent === null) return;@@ -138,19 +120,6 @@ function detach(node: HostNode): void {syncChars(parent);}-/**- * Cuts the links inside a subtree about to be destroyed, so whatever the- * reconciler still holds out of it reads as detached — the way its entities- * read as destroyed.- */-function detachSubtree(node: SceneNode): void {- for (const child of node.children) {- if (isSceneNode(child)) detachSubtree(child);- child.parent = null;- }- node.children.length = 0;-}-const TIME_TRAITS = {start: Start,end: End,@@ -164,6 +133,7 @@ const PAINT_TYPES: Record<string, PaintType> = {radialGradientPaint: PaintType.RADIAL_GRADIENT,shaderPaint: PaintType.SHADER,surfacePaint: PaintType.SURFACE,+ htmlPaint: PaintType.HTML,imagePaint: PaintType.IMAGE,videoPaint: PaintType.VIDEO,};@@ -365,29 +335,6 @@ const FONT_WEIGHTS: Record<string, string> = {bold: '700',};-/**- * Allocates a surface paint's host with the element (a canvas needs a DOM;- * a host without one leaves the paint blank and `canvas` null).- */-function attachSurfaceHost(paint: Entity): void {- if (typeof document === 'undefined') return;- paint.add(SurfaceHostHandle);- paint.set(SurfaceHostHandle, new SurfaceHost());-}--/**- * Sizes the surface canvases of `holder` (its own, for a `<surface>`, and- * those of its `<surfacePaint>` children) to its authored box, so they draw- * at the box's resolution. Same-size calls are no-ops (SurfaceHost).- */-function syncSurfaceSize(world: World, holder: Entity): void {- const size = holder.get(Size);- if (size === undefined) return;- holder.get(SurfaceHostHandle)?.setSize(size.width, size.height);- for (const child of getEntityChildren(world, holder)) {- child.get(SurfaceHostHandle)?.setSize(size.width, size.height);- }-}/*** A numeric prop's value, or undefined for none. A boolean is none, not 0@@ -411,14 +358,22 @@ function toSeconds(value: unknown): number | undefined {return parseTime(value);}-export class RuntimeDocument implements ProjectDocument<HostNode> {+export class RuntimeDocument implements ProjectDocument<SceneNode> {public readonly stage: SceneNode;private readonly world: World;public constructor(world: World) {this.world = world;const root = world.get(Root)!;- this.stage = { entity: root, tag: '', props: {}, parent: null, children: [] };+ this.stage = {+ entity: root,+ native: true,+ tag: '',+ props: {},+ parent: null,+ children: [],+ domNode: null,+ };root.add(Host);root.set(Host, this.stage);}@@ -434,18 +389,55 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {const held = entity.get(Host);if (held) return held;- const node: SceneNode = { entity, tag: '', props: {}, parent: null, children: [] };+ const node: SceneNode = {+ entity,+ native: true,+ tag: '',+ props: {},+ parent: null,+ children: [],+ domNode: null,+ };entity.add(Host);entity.set(Host, node);return node;}- public createElement(tag: string): SceneNode | SurfaceNode {- // Composition elements arrive in either spelling: the camelCase- // intrinsics a project authors, or the PascalCase components the compile- // step (and an app inserting elements) uses.+ public createElement(tag: string): SceneNode {+ // camelCase composition tags are DOM elements.+ if (tag.charAt(0) === tag.charAt(0).toLowerCase()) {+ if (typeof document === 'undefined') {+ throw new Error(`<${tag}> requires a DOM`);+ }+ if (tag === 'audio' || tag === 'video') {+ throw new Error(+ `<${tag}> is not supported as HTML content; use the <${tag}> composition element outside the HTML subtree`,+ );+ }+ if (tag === 'canvas') {+ throw new Error('<canvas> is not supported as HTML content; use <surface> for custom drawing');+ }+ const domNode = SVGElements.has(tag)+ ? document.createElementNS('http://www.w3.org/2000/svg', tag)+ : document.createElement(tag);+ const entity = createEntity(this.world);+ const node: SceneNode = {+ entity,+ native: false,+ tag,+ props: {},+ parent: null,+ children: [],+ domNode,+ };+ entity.add(Host);+ entity.set(Host, node);+ return node;+ }+const name = tag.charAt(0).toLowerCase() + tag.slice(1);let entity: Entity;+ let domNode: Element | null = null;switch (name) {case 'stage':@@ -561,6 +553,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {case 'radialGradientPaint':case 'shaderPaint':case 'surfacePaint':+ case 'htmlPaint':case 'imagePaint':case 'videoPaint': {entity = createEntity(this.world);@@ -568,7 +561,42 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.set(Paint, { value: PAINT_TYPES[name]! });if (name === 'solidPaint') entity.add(Color);if (name === 'shaderPaint') entity.add(Shader);- if (name === 'surfacePaint') attachSurfaceHost(entity);+ if (name === 'surfacePaint') {+ domNode = document.createElement('canvas');+ }+ if (name === 'htmlPaint') {+ const root = document.createElement('div');+ root.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';++ const canvas = this.world.get(RenderSurface)?.canvas;+ assert(canvas instanceof HTMLCanvasElement, 'HTML paint must be on a canvas');++ canvas.toggleAttribute('layoutsubtree', true);+ canvas.appendChild(root);++ domNode = root;+ }+ break;+ }+ case 'html': {+ entity = createEntity(this.world);+ entity.add(Geometry);+ entity.set(Geometry, { value: GeometryType.RECT });+ entity.add(Position);+ entity.set(Position, { x: 0, y: 0 });+ entity.add(Paint);+ entity.set(Paint, { value: PaintType.HTML });++ const root = document.createElement('div');+ root.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';+ const canvas = this.world.get(RenderSurface)?.canvas;+ assert(canvas instanceof HTMLCanvasElement, 'HTML paint must be on a canvas');++ canvas.toggleAttribute('layoutsubtree', true);+ canvas.appendChild(root);++ domNode = root;+ resizeEntity(this.world, entity, { width: 100, height: 100 });break;}case 'surface': {@@ -581,9 +609,8 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.set(Position, { x: 0, y: 0 });entity.add(Paint);entity.set(Paint, { value: PaintType.SURFACE });- attachSurfaceHost(entity);+ domNode = document.createElement('canvas');resizeEntity(this.world, entity, { width: 100, height: 100 });- syncSurfaceSize(this.world, entity);break;}case 'colorStop': {@@ -633,7 +660,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {}default:throw new Error(- `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <captions>, <adjustmentLayer>, <rect>, <text>, <textRange>, <video>, <image>, <audio>, <surface>, <solidPaint>, <linearGradientPaint>, <radialGradientPaint>, <shaderPaint>, <surfacePaint>, <imagePaint>, <videoPaint>, <colorStop>, <stroke>, <shadow>, <effect>, <animation>, <keyframeTrack> and <keyframe>).`,+ `<${tag}> is not supported yet (only composition elements exported by @diffusionstudio/jsx are accepted).`,);}@@ -645,18 +672,20 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {const node: SceneNode = {entity,+ native: true,tag: name,props: {},parent: null,- children: []+ children: [],+ domNode,};if (name === 'surface' || name === 'surfacePaint') {Object.assign(node, {get canvas() {- return entity.get(SurfaceHostHandle)?.canvas ?? null;+ return node.domNode instanceof HTMLCanvasElement ? node.domNode : null;},- })+ });}entity.add(Host);@@ -664,20 +693,31 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {return node;}- public createTextNode(text: string): TextNode {- return { text, parent: null };+ public createTextNode(text: string): SceneNode {+ const entity = createEntity(this.world);+ const node: SceneNode = {+ entity,+ native: false,+ tag: '#text',+ props: {},+ parent: null,+ children: [],+ domNode: document.createTextNode(text),+ };+ entity.add(Host);+ entity.set(Host, node);+ return node;}- public replaceText(node: HostNode, text: string): void {- if (isSceneNode(node)) return;- node.text = text;- if (node.parent) {- syncChars(node.parent);- }+ public replaceText(node: SceneNode, text: string): void {+ if (!isDomText(node)) return;+ node.domNode.data = text;+ const owner = node.parent;+ if (owner) syncChars(owner);}- public isTextNode(node: HostNode): boolean {- return !isSceneNode(node);+ public isTextNode(node: SceneNode): boolean {+ return isDomText(node);}/**@@ -699,13 +739,18 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {}for (const [index, part] of parts.entries()) {- part.text = index === 0 ? text : '';+ part.domNode.data = index === 0 ? text : '';}syncChars(node);}- public setProperty(node: HostNode, name: string, value: unknown): void {+ public setProperty(node: SceneNode, name: string, value: unknown): void {if (!isSceneNode(node)) return;++ if (!node.native && node.domNode instanceof Element) {+ setDomProperty(node.domNode, name, value);+ }+const { entity } = node;// The stage and anything adopted from outside are no element of the@@ -932,7 +977,6 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {return;}resizeEntity(this.world, entity, { [name]: size });- syncSurfaceSize(this.world, entity);return;}case 'join': {@@ -1368,21 +1412,64 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {* next value takes its place. Only a `<text>` reads its text children back* out as `Chars`.*/- public insertNode(parent: HostNode, node: HostNode, anchor?: HostNode): void {- if (!isSceneNode(parent)) {+ public insertNode(parent: SceneNode, node: SceneNode, anchor?: SceneNode): void {+ if (isDomText(parent)) {throw new Error('Text cannot contain children.');}- if (isSceneNode(node)) {- if (parent.entity === node.entity) return;- // Checked before anything moves: `appendChild` asserts the same- // thing, but only once the node has left the parent it had.- if (- getParentEntity(node.entity) !== parent.entity &&- getEntityTree(this.world, node.entity).includes(parent.entity)- ) {- throw new Error('Cannot parent entity into its own subtree');+ const candidate = (!parent.native || parent.tag === 'html' || parent.tag === 'htmlPaint')+ ? parent.domNode+ : null;+ const domContainer = candidate instanceof Element ? candidate : null;++ if (isDomText(node)) {+ detach(node);+ const at = anchor === undefined ? -1 : parent.children.indexOf(anchor);+ if (at === -1) parent.children.push(node);+ else parent.children.splice(at, 0, node);+ node.parent = parent;++ if (domContainer) {+ const domAnchor = anchor?.domNode;+ domContainer.insertBefore(+ node.domNode,+ domAnchor?.parentNode === domContainer ? domAnchor : null,+ );+ } else {+ node.domNode.remove();+ }+ syncChars(parent);+ return;+ }++ if (domContainer) {+ if (node.native || !(node.domNode instanceof Element)) {+ throw new Error('A composition element cannot be HTML content.');}++ detach(node);+ const at = anchor === undefined ? -1 : parent.children.indexOf(anchor);+ if (at === -1) parent.children.push(node);+ else parent.children.splice(at, 0, node);+ node.parent = parent;++ const domAnchor = anchor?.domNode;+ domContainer.insertBefore(node.domNode, domAnchor?.parentNode === domContainer ? domAnchor : null);+ return;+ }++ if (!node.native) {+ throw new Error('DOM content must be inside <html> or <htmlPaint>.');+ }++ if (parent.entity === node.entity) return;+ // Checked before anything moves: `appendChild` asserts the same+ // thing, but only once the node has left the parent it had.+ if (+ getParentEntity(node.entity) !== parent.entity &&+ getEntityTree(this.world, node.entity).includes(parent.entity)+ ) {+ throw new Error('Cannot parent entity into its own subtree');}detach(node);@@ -1391,11 +1478,6 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {else parent.children.splice(at, 0, node);node.parent = parent;- if (!isSceneNode(node)) {- syncChars(parent);- return;- }-if (getParentEntity(node.entity) !== parent.entity) {// appendChild only takes top-level entities, so a move between two// parents goes back through the stage on the way.@@ -1410,29 +1492,37 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {if (node.entity.has(KeyframeTrack)) {this.resolveTrackProperty(node.entity, node.props.property);}- if (node.entity.has(SurfaceHostHandle)) {- syncSurfaceSize(this.world, parent.entity);- }}let index = 0;for (const sibling of parent.children) {- if (!isSceneNode(sibling)) continue;+ if (!sibling.native) continue;sibling.entity.add(ItemIndex);sibling.entity.set(ItemIndex, { value: index++ });}}+ private destroySubtree(node: SceneNode): void {+ for (const child of [...node.children]) {+ this.destroySubtree(child);+ }+ node.children.length = 0;+ node.parent = null;+ node.domNode?.remove();++ if (node.entity.isAlive() && node.entity !== this.stage.entity) {+ node.entity.destroy();+ }+ }+/*** Takes `node` out of the document. The parent it comes out of is the one* it has rather than the one the caller names: a move has already put it* in the new one by the time the renderer asks the old one to drop it.*/- public removeNode(_parent: HostNode, node: HostNode): void {+ public removeNode(_parent: SceneNode, node: SceneNode): void {detach(node);- if (!isSceneNode(node)) return;-if (node.entity.has(Stage)) {for (const child of [...node.children]) {this.removeNode(node, child);@@ -1442,23 +1532,18 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {return;}- // Destroy cascades through the subtree; whatever the reconciler still- // holds out of it — a text node, a node a ref kept — learns it has no- // parent, the way the entities learn they are gone.- detachSubtree(node);- // ChildOf auto-destroys orphans, so paints go with it.- node.entity.destroy();+ this.destroySubtree(node);}- public getParentNode(node: HostNode): SceneNode | undefined {+ public getParentNode(node: SceneNode): SceneNode | undefined {return node.parent ?? undefined;}- public getFirstChild(node: HostNode): HostNode | undefined {- return isSceneNode(node) ? node.children[0] : undefined;+ public getFirstChild(node: SceneNode): SceneNode | undefined {+ return node.children[0];}- public getNextSibling(node: HostNode): HostNode | undefined {+ public getNextSibling(node: SceneNode): SceneNode | undefined {const parent = node.parent;if (parent === null) return undefined;const at = parent.children.indexOf(node);@@ -1467,7 +1552,18 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {public dispose(): void {- this.removeNode(this.stage, this.stage);+ for (const entity of this.world.entities) {+ if (!entity.isAlive() || entity === this.stage.entity) continue;+ const domNode = entity.get(Host)?.domNode;+ if (domNode instanceof HTMLImageElement && domNode.src.startsWith('blob:')) {+ URL.revokeObjectURL(domNode.src);+ }+ if (domNode) {+ domNode.remove();+ }+ entity.destroy();+ }+// The world is free for the next mount. A document already replaced by// another leaves the registry as it found it.if (documents.get(this.world) === this) {@@ -1476,6 +1572,65 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {}}+/**+ * Prop assignment for real DOM nodes below an HTML paint. This mirrors the+ * useful, non-interactive part of Solid's DOM conventions: reactive styles,+ * classes, attributes and explicit HTML/text content. Pointer and media event+ * handlers are ignored because the subtree is rasterized into the canvas.+ */+function setDomProperty(element: Element | null, name: string, value: unknown): void {+ if (+ typeof value === 'function'+ || name.startsWith('on')+ || element === null+ || name === 'children'+ || name === 'ref'+ ) return;++ if (name === 'style') {+ const style = (element as HTMLElement | SVGElement).style;+ if (typeof value === 'string') {+ style.cssText = value;+ } else if (typeof value === 'object' && value !== null) {+ Object.assign(style, value);+ } else {+ style.cssText = '';+ }+ return;+ }++ if (name === 'class' || name === 'className') {+ if (value === undefined || value === null || value === false) element.removeAttribute('class');+ else element.setAttribute('class', String(value));+ return;+ }++ if (name === 'classList') {+ if (typeof value === 'object' && value !== null) {+ for (const [key, enabled] of Object.entries(value)) {+ element.classList.toggle(key, Boolean(enabled));+ }+ }+ return;+ }++ if (name === 'innerHTML') {+ element.innerHTML = value === undefined || value === null ? '' : String(value);+ return;+ }++ if (name === 'textContent') {+ element.textContent = value === undefined || value === null ? '' : String(value);+ return;+ }++ if (value === undefined || value === null || value === false) {+ element.removeAttribute(name);+ } else {+ element.setAttribute(name, value === true ? '' : String(value));+ }+}+const documents = new WeakMap<World, RuntimeDocument>();export function getRuntimeDocument(world: World): RuntimeDocument {@@ -1504,3 +1659,9 @@ export function createRuntimeDocument(world: World): RuntimeDocument {documents.set(world, document);return document;}++function assert(condition: unknown, message: string): asserts condition {+ if (!condition) {+ throw new Error(message);+ }+}diff --git a/packages/reconciler/src/elements.ts b/packages/reconciler/src/elements.tsindex 9a1eb002..64c59458 100644--- a/packages/reconciler/src/elements.ts+++ b/packages/reconciler/src/elements.ts@@ -39,6 +39,7 @@ import type {ShaderPaintProps,ShadowProps,SolidPaintProps,+ StageProps,StrokeProps,SurfacePaintProps,SurfaceProps,@@ -71,7 +72,7 @@ function hostElement<P extends object>(tag: string): (props: P) => SolidJSX.Elem* editor renders it inside `insertElement`'s thunk to duplicate a subtree.*/export function renderAuthored(tree: AuthoredTree): unknown {- const node = createElement(tree.tag);+ const node = createElement(tree.tag.charAt(0).toUpperCase() + tree.tag.slice(1));for (const [name, value] of Object.entries(tree.props)) {setProp(node, name, value);}@@ -84,6 +85,7 @@ export function renderAuthored(tree: AuthoredTree): unknown {return node;}+export const Stage = hostElement<StageProps>("Stage");export const Scene = hostElement<SceneProps>("Scene");export const Group = hostElement<GroupProps>("Group");export const Rect = hostElement<RectProps>("Rect");diff --git a/packages/runtime/src/actions/resize.ts b/packages/runtime/src/actions/resize.tsindex c81dc327..3d48e545 100644--- a/packages/runtime/src/actions/resize.ts+++ b/packages/runtime/src/actions/resize.ts@@ -9,6 +9,7 @@ import { ConstraintType } from '../constants';import {ChildOf, Group, Size, Position, Constraint, ConstraintCache,KeepAspectRatio, Sequential, Computed,+ Host,} from '../traits';import { getParentNode } from '../queries/hierarchy';@@ -57,6 +58,20 @@ export function resizeEntity(world: World, entity: Entity, params: ResizeParams)...(width !== undefined && { width: Math.round(width) }),...(height !== undefined && { height: Math.round(height) }),});++ // handle DOM nodes+ const domNode = entity.get(Host)?.domNode;++ if (domNode instanceof HTMLCanvasElement) {+ domNode.width = width ?? domNode.width;+ domNode.height = height ?? domNode.height;+ }++ if (domNode instanceof HTMLElement) {+ domNode.style.width = `${width ?? domNode.style.width.replace('px', '')}px`;+ domNode.style.height = `${height ?? domNode.style.height.replace('px', '')}px`;+ }+// The Size observer propagates Computed sizes; constraints resolve here.resolveConstraintOffsets(world, entity);}diff --git a/packages/runtime/src/media/html.ts b/packages/runtime/src/media/html.tsindex ec4de380..0738fd8d 100644--- a/packages/runtime/src/media/html.ts+++ b/packages/runtime/src/media/html.ts@@ -2,15 +2,6 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;--// html-in-canvas (https://github.com/WICG/html-in-canvas). Chromium only,-// behind chrome://flags/#canvas-draw-element; the API surface is still-// moving, so every touchpoint is typed and isolated here.-type DrawElementContext = Ctx2D & {- drawElementImage(source: Element | unknown, dx: number, dy: number, dw: number, dh: number): DOMMatrix;-};-export function isHtmlInCanvasSupported(): boolean {return typeof CanvasRenderingContext2D !== 'undefined'&& 'drawElementImage' in CanvasRenderingContext2D.prototype;@@ -31,122 +22,18 @@ export function nextRenderingUpdate(): Promise<void> {});}-let htmlCanvas: HTMLCanvasElement | null = null;-let htmlCtx: CanvasRenderingContext2D | null = null;-let htmlHosts = 0;--function acquireLayoutCanvas(): HTMLCanvasElement {- if (!htmlCanvas) {- htmlCanvas = document.createElement('canvas');- htmlCanvas.width = 1;- htmlCanvas.height = 1;- htmlCanvas.toggleAttribute('layoutsubtree', true);- htmlCanvas.style.cssText = 'position:fixed;left:0;top:0;width:1px;height:1px;pointer-events:none;';- document.body.appendChild(htmlCanvas);- htmlCtx = htmlCanvas.getContext('2d')!;- }- htmlHosts++;- return htmlCanvas;-}--function releaseLayoutCanvas(): void {- if (--htmlHosts === 0) {- htmlCanvas?.remove();- htmlCanvas = null;- htmlCtx = null;- }-}--/**- * DOM host for one HTML paint: a div inside the shared layoutsubtree canvas.- * The mount document inserts the paint's JSX children into `element` as real,- * reactive DOM; the browser lays them out at the parent geometry's box size- * and `draw` rasterizes the result at the destination's device scale and- * blits it into the rendering context (stage or offscreen alike).- */-export class HtmlHost {- public readonly element: HTMLDivElement;- private disposed = false;- private warned = false;- private width = -1;- private height = -1;-- public constructor() {- const el = document.createElement('div');- // pointer-events: none — layoutsubtree children participate in hit- // testing and would otherwise swallow page interactions.- el.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';- acquireLayoutCanvas().appendChild(el);- this.element = el;+/** Resolve asynchronously loaded resources below an HTML paint root. */+export function whenHtmlReady(root: HTMLElement, timeInSeconds: number): Promise<void> {+ for (const animation of root.getAnimations({ subtree: true })) {+ if (animation.playState !== 'paused') animation.pause();+ animation.currentTime = timeInSeconds * 1000;}- /**- * Resolves asynchronously loaded html resources.- */- public whenReady(timeInSeconds: number): Promise<void> {- // sync animations- for (const animation of this.element.getAnimations({ subtree: true })) {- if (animation.playState !== 'paused') animation.pause();- animation.currentTime = timeInSeconds * 1000;- }-- // wait for fonts to be ready- const pending: Promise<unknown>[] = [document.fonts.ready];-- // wait for images to be ready- for (const image of this.element.querySelectorAll('img')) {- // `complete` covers loaded, failed, and srcless images alike; the- // rest are still in flight, and a failed decode just paints nothing.- if (image.complete) continue;- pending.push(image.decode().catch(() => undefined));- }-- return Promise.all(pending).then(() => nextRenderingUpdate());- }-- public setSize(width: number, height: number): void {- if (width === this.width && height === this.height) return;- this.width = width;- this.height = height;- this.element.style.width = `${width}px`;- this.element.style.height = `${height}px`;- }-- public draw(ctx: Ctx2D, width: number, height: number): void {- if (this.disposed || !htmlCanvas || !htmlCtx) return;-- this.setSize(width, height);-- try {- const m = ctx.getTransform();- const sx = Math.hypot(m.a, m.b) || 1;- const sy = Math.hypot(m.c, m.d) || 1;- const pw = Math.max(1, Math.ceil(width * sx));- const ph = Math.max(1, Math.ceil(height * sy));-- // Grow-only: shrinking would clear and reallocate every time hosts- // of different sizes share the canvas within one frame.- if (htmlCanvas.width < pw) htmlCanvas.width = pw;- if (htmlCanvas.height < ph) htmlCanvas.height = ph;-- htmlCtx.setTransform(sx, 0, 0, sy, 0, 0);- htmlCtx.clearRect(0, 0, width, height);- (htmlCtx as DrawElementContext).drawElementImage(this.element, 0, 0, width, height);- ctx.drawImage(htmlCanvas, 0, 0, pw, ph, 0, 0, width, height);- } catch (e) {- // A transient failure (subtree not yet laid out or painted) just- // skips the frame; log once so a permanent one doesn't spam.- if (!this.warned) {- this.warned = true;- console.error(`Error drawing <HtmlPaint> content: ${e instanceof Error ? `${e.name}: ${e.message}` : e}`);- }- }+ const pending: Promise<unknown>[] = [document.fonts.ready];+ for (const image of root.querySelectorAll('img')) {+ if (image.complete) continue;+ pending.push(image.decode().catch(() => undefined));}- public dispose(): void {- if (this.disposed) return;- this.disposed = true;- this.element.remove();- releaseLayoutCanvas();- }+ return Promise.all(pending).then(() => nextRenderingUpdate());}diff --git a/packages/runtime/src/media/index.ts b/packages/runtime/src/media/index.tsindex 61a4d715..b30f5a43 100644--- a/packages/runtime/src/media/index.ts+++ b/packages/runtime/src/media/index.ts@@ -15,7 +15,6 @@ export * from './image';export * from './keyframe-index';export * from './sequence';export * from './shader';-export * from './surface';export * from './time-stretcher';export * from './transcode';export * from './video';diff --git a/packages/runtime/src/media/surface.ts b/packages/runtime/src/media/surface.tsdeleted file mode 100644index bd31a790..00000000--- a/packages/runtime/src/media/surface.ts+++ /dev/null@@ -1,44 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;--/**- * Backing store for one surface paint: a detached canvas handed to the- * mount's `ref` callback, which draws into it with any context type (2d,- * webgl, webgpu). The engine samples the bitmap every frame and stretches it- * into the parent geometry's box. The host allocates the bitmap at the box- * size and follows the box's authored size; user code (e.g. three.js via- * `renderer.setSize`) may resize it through its own APIs as well.- */-export class SurfaceHost {- public readonly canvas = document.createElement('canvas');- private disposed = false;-- /**- * Bitmap allocation at the box size. A no-op when the size is unchanged:- * assigning a canvas dimension clears the bitmap even when the value is- * the same, and user code may already have applied this size itself.- */- public setSize(width: number, height: number): void {- const w = Math.max(1, Math.round(width));- const h = Math.max(1, Math.round(height));- if (this.canvas.width !== w) this.canvas.width = w;- if (this.canvas.height !== h) this.canvas.height = h;- }-- public draw(ctx: Ctx2D, width: number, height: number): void {- if (this.disposed || this.canvas.width === 0 || this.canvas.height === 0) return;- ctx.drawImage(this.canvas, 0, 0, width, height);- }-- public dispose(): void {- if (this.disposed) return;- this.disposed = true;- // Release the backing store; context teardown (e.g. a WebGL renderer)- // is the mount's own onCleanup concern.- this.canvas.width = 0;- this.canvas.height = 0;- }-}diff --git a/packages/runtime/src/queries/predicates.ts b/packages/runtime/src/queries/predicates.tsindex e54860ef..a7703300 100644--- a/packages/runtime/src/queries/predicates.ts+++ b/packages/runtime/src/queries/predicates.ts@@ -64,9 +64,11 @@ export function isMask(entity: Entity): boolean {}export function hasHtmlPaint(entity: Entity): boolean {- return (entity.get(Cache)?.fills ?? []).some(fill => fill.get(Paint)?.value === PaintType.HTML);+ return entity.get(Paint)?.value === PaintType.HTML+ || (entity.get(Cache)?.fills ?? []).some(fill => fill.get(Paint)?.value === PaintType.HTML);}export function hasSurfacePaint(entity: Entity): boolean {- return (entity.get(Cache)?.fills ?? []).some(fill => fill.get(Paint)?.value === PaintType.SURFACE);+ return entity.get(Paint)?.value === PaintType.SURFACE+ || (entity.get(Cache)?.fills ?? []).some(fill => fill.get(Paint)?.value === PaintType.SURFACE);}diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsindex 675a7918..cf88620e 100644--- a/packages/runtime/src/systems/assets.ts+++ b/packages/runtime/src/systems/assets.ts@@ -2,14 +2,23 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Ai, FramePromises, Generating, GenerationRequest, Library, LoadRequest, PendingSource, SourceError, TranscriptionRequest } from '../traits';-import { bindAsset, getModifiers } from '../actions/assets';+import { Ai, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, SourceError, TranscriptionRequest } from '../traits';+import { bindAsset, getAssetFile, getModifiers } from '../actions/assets';import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';import type { Entity, World } from 'koota';import type { Asset } from '@diffusionstudio/assets';export function assetSystem(world: World): void {+ for (const entity of world.query(LoadRequest, Host)) {+ const source = entity.get(LoadRequest)!.value;+ if (!isDomImage(entity)) continue;+ pointDomImageAt(entity, null);+ if (source !== '' && !/^(?:data|blob):/i.test(source)) continue;+ entity.remove(LoadRequest);+ pointDomImageAt(entity, source || null);+ }+const library = world.get(Library);if (library) {for (const entity of world.query(LoadRequest)) {@@ -25,6 +34,7 @@ export function assetSystem(world: World): void {const ref = entity.get(GenerationRequest)!.ref;entity.remove(GenerationRequest);if (ref === null || entity.has(SourceError)) continue;+ if (isDomImage(entity)) pointDomImageAt(entity, null);start(world, entity, ai.resolve(ref), true);}@@ -88,24 +98,59 @@ function resolve(world: World, entity: Entity, promise: Promise<Asset>, generatientity.set(PendingSource, { value: token });if (generating) entity.add(Generating);- const done = promise.then(- (asset) => {- if (!current(entity, token)) return;- entity.remove(PendingSource, Generating);- bindAsset(entity, asset);- },- (error: unknown) => {- if (!current(entity, token)) return;- entity.remove(PendingSource, Generating);- entity.add(SourceError);- entity.set(SourceError, { value: errorMessage(error), generated: generating });- console.error('[runtime] could not resolve src:', error);- },- );+ const done = promise.then(async (asset) => {+ if (!current(entity, token)) return;+ bindAsset(entity, asset);+ await bindDomImage(entity, asset, token);+ if (current(entity, token)) entity.remove(PendingSource, Generating);+ }).catch((error: unknown) => {+ if (!current(entity, token)) return;+ entity.remove(PendingSource, Generating);+ entity.add(SourceError);+ entity.set(SourceError, { value: errorMessage(error), generated: generating });+ console.error('[runtime] could not resolve src:', error);+ });world.get(FramePromises)?.list?.push(done);}+function isDomImage(entity: Entity): boolean {+ return typeof HTMLImageElement !== 'undefined'+ && entity.get(Host)?.domNode instanceof HTMLImageElement;+}++function pointDomImageAt(entity: Entity, source: string | null): void {+ const node = entity.get(Host);+ const image = node?.domNode;+ if (typeof HTMLImageElement === 'undefined' || !(image instanceof HTMLImageElement) || !node) return;++ releaseDomImageSource(image);+ if (source === null) image.removeAttribute('src');+ else image.src = source;+}++export function releaseDomImageSource(image: HTMLImageElement): void {+ if (image.src.startsWith('blob:')) URL.revokeObjectURL(image.src);+}++async function bindDomImage(entity: Entity, asset: Asset, token: object): Promise<void> {+ if (!isDomImage(entity)) return;+ const file = await getAssetFile(asset);+ if (!current(entity, token)) return;++ const url = URL.createObjectURL(file);+ if (!current(entity, token)) {+ URL.revokeObjectURL(url);+ return;+ }++ pointDomImageAt(entity, url);+ const image = entity.get(Host)?.domNode;+ if (typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement) {+ await image.decode().catch(() => undefined);+ }+}+/** What a rejection says, for an entity to carry and the host to show. */function errorMessage(error: unknown): string {const message = error instanceof Error ? error.message : String(error);diff --git a/packages/runtime/src/systems/playback.ts b/packages/runtime/src/systems/playback.tsindex 3b9f4c15..8ce2699d 100644--- a/packages/runtime/src/systems/playback.ts+++ b/packages/runtime/src/systems/playback.ts@@ -16,7 +16,7 @@ import {Geometry, Group, AdjustmentLayer, Paint, Audio, Caption, Muted, Soloed,Sequential, Transition, Playback, Workarea,AudioPlayback, Computed,- AudioDecoderHandle, AudioBusHandle, HtmlHostHandle,+ AudioDecoderHandle, AudioBusHandle, Host,Mode, FrameRate, Time, AudioEngine, FramePromises, Mounts,Root,} from '../traits';@@ -28,6 +28,7 @@ import {resolveAudioDecoder, resolveCaptionDecoder, resolveImageDecoder,resolveShaderHost, resolveVideoDecoder,} from '../media';+import { whenHtmlReady } from '../media/html';import { AudioBus } from '../media/audio-bus';import type { Entity, World } from 'koota';@@ -201,9 +202,11 @@ function forwardAudioDecoder(world: World, scene: Entity, entity: Entity, audioSfunction forwardHtmlHost(world: World, scene: Entity, entity: Entity, fill: Entity): void {const computed = store(world, Computed);- const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;- if (world.get(Mode)?.value === 'realtime' || computed.visibility[entity.id()] !== 1 || !host) return;- framePromises(world)?.push(host.whenReady(computed.localTimeInSeconds[scene.id()] ?? 0));+ const root = fill.get(Host)?.domNode;+ if (world.get(Mode)?.value === 'realtime'+ || computed.visibility[entity.id()] !== 1+ || !(root instanceof HTMLElement)) return;+ framePromises(world)?.push(whenHtmlReady(root, computed.localTimeInSeconds[scene.id()] ?? 0));}function forwardImageDecoder(world: World, _scene: Entity, _entity: Entity, fill: Entity): void {@@ -238,6 +241,8 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {intrinsicVideo = true;} else if (intrinsic === PaintType.IMAGE && visualsEnabled) {forwardImageDecoder(world, scene, entity, entity);+ } else if (intrinsic === PaintType.HTML && visualsEnabled) {+ forwardHtmlHost(world, scene, entity, entity);}for (const fill of world.query(ChildOf(entity), Paint, Not(Geometry), Not(Hidden))) {diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex ff68a6d1..6230d1d8 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -3,8 +3,8 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */// Render system (was systems/render.ts): draws the document tree onto the-// world's RenderSurface. Runs identically against the editor canvas and an-// OffscreenCanvas during capture; without a surface it is a no-op. Hit+// world's RenderSurface. Runs identically against the editor and capture+// canvases; without a surface it is a no-op. Hit// regions are pushed callback-less (see HitRegions); the app's input layer// attaches its handlers.@@ -21,7 +21,7 @@ import {SourceError,BlendMode, Effect, Transition, MixedCornerRadius,LocalTransform, WorldTransform, Computed, Cache,- HtmlHostHandle, SurfaceHostHandle,+ Host,Mode, FrameRate, Camera, Background, RenderSurface,HitRegions,Root,@@ -215,18 +215,22 @@ export function renderIntrinsicFill(world: World, entity: Entity): void {const intrinsic = getIntrinsicPaint(entity);if (intrinsic === PaintType.SURFACE) {- const host = entity.has(SurfaceHostHandle) ? entity.get(SurfaceHostHandle) : null;- if (host) {+ const canvas = entity.get(Host)?.domNode;+ if (canvas instanceof HTMLCanvasElement) {const ctx = getCtx(world);const computed = store(world, Computed);const eid = entity.id();ctx.save();ctx.clip();- host.draw(ctx, computed.width[eid]!, computed.height[eid]!);+ ctx.drawImage(canvas, 0, 0, computed.width[eid]!, computed.height[eid]!);ctx.restore();}return;}+ if (intrinsic === PaintType.HTML) {+ renderHtmlFill(world, entity, entity);+ return;+ }const kind = mediaKind(intrinsic);if (kind === null) return;@@ -245,6 +249,41 @@ function mediaKind(paint: PaintType | undefined): MediaKind | null {/** What the image and video decoders hand out to draw. */type MediaFrame = ImageBitmap | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas;+// html-in-canvas (https://github.com/WICG/html-in-canvas). Chromium only,+// behind chrome://flags/#canvas-draw-element; the API surface is still+// moving, so every touchpoint is typed and isolated here.+type DrawElementContext = Ctx2D & {+ drawElementImage(source: Element | unknown, dx: number, dy: number, dw: number, dh: number): DOMMatrix;+};+++function renderHtmlFill(world: World, entity: Entity, source: Entity): void {+ const root = source.get(Host)?.domNode;+ const surface = world.get(RenderSurface);+ const ctx = surface?.ctx;+ if (!(ctx instanceof CanvasRenderingContext2D)) return;+ if (!(root instanceof HTMLElement)) return;++ const computed = store(world, Computed);+ const eid = entity.id();++ ctx.save();+ ctx.clip();++ const width = computed.width[eid]!;+ const height = computed.height[eid]!;+ root.style.width = `${width}px`;+ root.style.height = `${height}px`;++ try {+ (ctx as DrawElementContext).drawElementImage(root, 0, 0, width, height);+ } catch (error) {+ console.error(`Error drawing <HtmlPaint> content: ${error instanceof Error ? `${error.name}: ${error.message}` : error}`);+ }++ ctx.restore();+}+/*** Draws the current frame of `source` (an image or video paint, or the* geometry itself for intrinsic media) into `entity`'s box, fitted by the@@ -312,21 +351,13 @@ export function renderFills(world: World, entity: Entity): void {} else if (paint === PaintType.VIDEO) {renderMedia(world, entity, fill, 'VIDEO');} else if (paint === PaintType.HTML) {- const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;-- if (host) {- ctx.save();- ctx.clip();- host.draw(ctx, computed.width[eid]!, computed.height[eid]!);- ctx.restore();- }+ renderHtmlFill(world, entity, fill);} else if (paint === PaintType.SURFACE) {- const host = fill.has(SurfaceHostHandle) ? fill.get(SurfaceHostHandle) : null;-- if (host) {+ const canvas = fill.get(Host)?.domNode;+ if (canvas instanceof HTMLCanvasElement) {ctx.save();ctx.clip();- host.draw(ctx, computed.width[eid]!, computed.height[eid]!);+ ctx.drawImage(canvas, 0, 0, computed.width[eid]!, computed.height[eid]!);ctx.restore();}} else if (paint === PaintType.SOLID) {diff --git a/packages/runtime/src/traits/document.ts b/packages/runtime/src/traits/document.tsnew file mode 100644index 00000000..d0d0bc3d--- /dev/null+++ b/packages/runtime/src/traits/document.ts@@ -0,0 +1,34 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { trait } from 'koota';++import type { Entity } from 'koota';++/** The real DOM node owned by a scene node. */+export type DomNode = Element | Text;++/**+ * One element of the document, and its place in it. `parent` and `children`+ * are the tree the reconciler reads back: text nodes and element nodes in the+ * one order they were inserted in, which is what the document answers+ * `getFirstChild` and `getNextSibling` from.+ */+export interface SceneNode {+ readonly entity: Entity;+ /** Native composition elements participate in the Koota scene graph. */+ readonly native: boolean;+ tag: string;+ props: Record<string, unknown>;+ parent: SceneNode | null;+ children: SceneNode[];+ readonly domNode: DomNode | null;+}++// The document node an entity was rendered from, on the entity itself: the+// one object the document and the renderer both hold for it, so `===` between+// two of them means what it says. AoS storage like the other handle traits —+// entity.get(Host) returns the node — and never serialized: a node is this+// render's business, rebuilt whenever a document renders the entity again.+export const Host = trait(() => null as SceneNode | null);diff --git a/packages/runtime/src/traits/index.ts b/packages/runtime/src/traits/index.tsindex 5553c0a2..e54f947b 100644--- a/packages/runtime/src/traits/index.ts+++ b/packages/runtime/src/traits/index.ts@@ -1,5 +1,6 @@export * from './relations';export * from './node';+export * from './document';export * from './transform';export * from './style';export * from './text';diff --git a/packages/runtime/src/traits/media.ts b/packages/runtime/src/traits/media.tsindex 80bd135d..4b9f85db 100644--- a/packages/runtime/src/traits/media.ts+++ b/packages/runtime/src/traits/media.ts@@ -4,10 +4,6 @@import { trait } from 'koota';-// Type-only imports: erased at compile time, so the traits ↔ media module-// cycle never materializes at runtime.-import type { HtmlHost } from '../media/html';-import type { SurfaceHost } from '../media/surface';import type { ShaderHost } from '../media/shader';import type { ImageDecoder } from '../media/image';import type { VideoDecoderInstance } from '../media/video';@@ -22,14 +18,9 @@ export type MountData = {textBox?: DocumentFragment;};-// Handle traits store live runtime instances (hosts, decoders, buses) and are+// Handle traits store live runtime instances (decoders, buses, etc.) and are// never serialized. They use AoS storage: entity.get(X) returns the instance-// directly. Suffixed Handle so the trait and the class it stores (HtmlHost,-// AudioDecoder, ...) can share one export surface.--export const HtmlHostHandle = trait(() => null as HtmlHost | null);--export const SurfaceHostHandle = trait(() => null as SurfaceHost | null);+// directly.export const ShaderHostHandle = trait(() => null as ShaderHost | null);diff --git a/packages/runtime/src/traits/world.ts b/packages/runtime/src/traits/world.tsindex edf7fac2..87d50b50 100644--- a/packages/runtime/src/traits/world.ts+++ b/packages/runtime/src/traits/world.ts@@ -48,8 +48,8 @@ export const Time = trait({ now: 0, delta: 0 });// Project frame rate (frames per second).export const FrameRate = trait({ value: 30 });-// Render target injected by the host: the editor canvas in the app, an-// OffscreenCanvas during capture. resolution is the device pixel ratio.+// Render target injected by the host. Capture uses an HTML canvas too so HTML+// paint roots can live in its layout subtree. resolution is the pixel ratio.export const RenderSurface = trait({canvas: () => null as HTMLCanvasElement | OffscreenCanvas | null,ctx: () => null as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null,diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.tsindex e48ac428..524694fe 100644--- a/packages/runtime/src/world/observers.ts+++ b/packages/runtime/src/world/observers.ts@@ -26,7 +26,7 @@ import {ColorStop, StrokeStyle, Size, Computed, Active, Stage, IsMask,ImageDecoderHandle, VideoDecoderHandle,AudioDecoderHandle, CaptionDecoderHandle, WaveformHandle,- HtmlHostHandle, SurfaceHostHandle, ShaderHostHandle, AudioBusHandle,+ ShaderHostHandle, AudioBusHandle,} from '../traits';import { getParentEntity } from '../queries/hierarchy';import { evictFromCaches, rebuildCaches, refileMask } from '../actions/cache';@@ -83,8 +83,6 @@ export function observeWorld(world: World): () => void {subs.push(world.onRemove(AudioDecoderHandle, (e) => e.get(AudioDecoderHandle)?.reset()));subs.push(world.onRemove(CaptionDecoderHandle, (e) => e.get(CaptionDecoderHandle)?.dispose()));subs.push(world.onRemove(WaveformHandle, (e) => e.get(WaveformHandle)?.dispose()));- subs.push(world.onRemove(HtmlHostHandle, (e) => e.get(HtmlHostHandle)?.dispose()));- subs.push(world.onRemove(SurfaceHostHandle, (e) => e.get(SurfaceHostHandle)?.dispose()));subs.push(world.onRemove(ShaderHostHandle, (e) => e.get(ShaderHostHandle)?.dispose()));subs.push(world.onRemove(AudioBusHandle, (e) => e.get(AudioBusHandle)?.disconnect()));