Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance JSX composition handling and improve DOM integration

konstantin-paulus2272d029ae59
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.ts
index c86ee4d1..0ec3820e 100644
--- a/apps/desktop/src/projects.ts
+++ b/apps/desktop/src/projects.ts
@@ -27,7 +27,7 @@ import type { BuildOptions, Plugin } from "esbuild";
import { mainBridge } from "./main-manager";
import { MAIN_CHANNELS } from "./main-channels";
import { applyEdits, stampProject } from "./edit";
-import { sourcePlugin } from "./source";
+import { canonicalizeTagsPlugin, sourcePlugin } from "./source";
import type { CompileResult, FsEntry, FsStat, ProjectInfo, SourceEdit, WriteResult } from "./main-channels";
import type { SourceContext } from "./edit";
@@ -615,7 +615,7 @@ const babelOptions = (file: string, filename: string): TransformOptions => ({
configFile: false,
// Runs before the presets, which is what it needs: Solid's transform
// replaces the JSX trees this stamps.
- plugins: [[sourcePlugin, { file }]],
+ plugins: [[sourcePlugin, { file }], canonicalizeTagsPlugin],
presets: [
[preset("babel-preset-solid"), { generate: "universal", moduleName: RUNTIME_MODULE }],
[preset("@babel/preset-typescript"), { onlyRemoveTypeImports: true }],
diff --git a/apps/desktop/src/source.ts b/apps/desktop/src/source.ts
index 991b6e0b..91caf47f 100644
--- a/apps/desktop/src/source.ts
+++ b/apps/desktop/src/source.ts
@@ -11,13 +11,113 @@
// disagreed, a drag would write to the wrong element; that invariant is why
// these two modules are read together.
-import { ID_ATTR, LOOP_ATTR, SOURCE_ATTR, formatSource, isCompositionTag, isLoopTag } from "@diffusionstudio/jsx";
+import { COMPOSITION_TAGS, ID_ATTR, LOOP_ATTR, SOURCE_ATTR, formatSource, isCompositionTag, isLoopTag } from "@diffusionstudio/jsx";
-import type { PluginObj, types as t } from "@babel/core";
+import type { NodePath, PluginObj, types as t } from "@babel/core";
/** What babel hands a plugin factory. */
type BabelApi = { types: typeof import("@babel/core").types };
+// Composition tags are authored in camelCase but must reach the renderer as
+// PascalCase components. Lowercase is reserved at runtime for real DOM nodes
+// inside <html>/<htmlPaint>.
+const COMPOSITION_COMPONENTS = new Map<string, string>(
+ COMPOSITION_TAGS.map((tag) => [tag, tag.charAt(0).toUpperCase() + tag.slice(1)]),
+);
+
+// These composition names also exist in SVG. An SVG fragment is DOM content
+// only while it has an SVG container in the same JSX tree; elsewhere the tag
+// keeps its composition meaning.
+const AMBIGUOUS_SVG_TAGS: ReadonlySet<string> = new Set(["rect", "text", "image"]);
+const SVG_CONTAINERS: ReadonlySet<string> = new Set([
+ "svg", "g", "defs", "symbol", "marker", "mask", "clipPath", "pattern",
+ "filter", "linearGradient", "radialGradient", "textPath", "tspan", "switch",
+]);
+
+const PASCAL_ELEMENTS = new Map(
+ [...COMPOSITION_COMPONENTS].map(([camel, pascal]) => [pascal, camel]),
+);
+
+const SOLID_CONTROL_FLOW: ReadonlySet<string> = new Set([
+ "For", "Show", "Switch", "Match", "Suspense", "SuspenseList", "Index", "ErrorBoundary",
+]);
+
+function hasSvgAncestor(path: NodePath<t.JSXElement>, types: typeof import("@babel/core").types): boolean {
+ return path.findParent(
+ (parent) => parent.isJSXElement()
+ && types.isJSXIdentifier(parent.node.openingElement.name)
+ && SVG_CONTAINERS.has(parent.node.openingElement.name.name),
+ ) !== null;
+}
+
+function isSvgCollision(path: NodePath<t.JSXElement>, name: string, types: typeof import("@babel/core").types): boolean {
+ return AMBIGUOUS_SVG_TAGS.has(name) && hasSvgAncestor(path, types);
+}
+
+/**
+ * Rewrites authored composition intrinsics to aliased PascalCase imports
+ * before babel-preset-solid consumes the JSX tree. Aliases prevent a user
+ * binding named e.g. `Rect` from capturing an authored `<rect>`.
+ */
+export function canonicalizeTagsPlugin({ types }: BabelApi): PluginObj {
+ return {
+ name: "jsx-canonical-composition-tags",
+ visitor: {
+ // Babel merges plugin and preset visitors. Rewriting the complete tree
+ // on Program enter ensures Solid cannot consume a parent before this
+ // pass reaches its descendants.
+ Program(program) {
+ const aliases = new Map<string, string>();
+
+ program.traverse({
+ JSXElement(path) {
+ const name = path.node.openingElement.name;
+ if (!types.isJSXIdentifier(name)) return;
+
+ if (/^[A-Z]/.test(name.name) && !path.scope.hasBinding(name.name)) {
+ const camel = PASCAL_ELEMENTS.get(name.name);
+ if (camel !== undefined) {
+ throw path.buildCodeFrameError(
+ `<${name.name}> is not a tag; write the composition element as <${camel}>`,
+ );
+ }
+ if (SOLID_CONTROL_FLOW.has(name.name)) {
+ throw path.buildCodeFrameError(
+ `<${name.name}> needs an import: add \`import { ${name.name} } from "solid-js"\``,
+ );
+ }
+ return;
+ }
+
+ const component = COMPOSITION_COMPONENTS.get(name.name);
+ if (component === undefined || isSvgCollision(path, name.name, types)) return;
+
+ let alias = aliases.get(component);
+ if (alias === undefined) {
+ alias = program.scope.generateUidIdentifier(component).name;
+ aliases.set(component, alias);
+ }
+
+ path.node.openingElement.name = types.jsxIdentifier(alias);
+ if (path.node.closingElement) {
+ path.node.closingElement.name = types.jsxIdentifier(alias);
+ }
+ },
+ });
+
+ if (aliases.size === 0) return;
+ const specifiers = [...aliases].map(([name, alias]) =>
+ types.importSpecifier(types.identifier(alias), types.identifier(name)),
+ );
+ program.unshiftContainer(
+ "body",
+ types.importDeclaration(specifiers, types.stringLiteral("@diffusionstudio/jsx")),
+ );
+ },
+ },
+ };
+}
+
const jsxTagName = (element: t.JSXElement): string | undefined => {
const name = element.openingElement.name;
return name.type === "JSXIdentifier" ? name.name : undefined;
@@ -51,7 +151,11 @@ export function sourcePlugin({ types }: BabelApi, { file }: { file: string }): P
const opening = path.node.openingElement;
const name = opening.name;
- if (name.type !== "JSXIdentifier" || !isCompositionTag(name.name)) return;
+ if (
+ name.type !== "JSXIdentifier"
+ || !isCompositionTag(name.name)
+ || isSvgCollision(path, name.name, types)
+ ) return;
const loop = path.findParent(
(parent) => parent.isJSXElement() && isLoopTag(jsxTagName(parent.node) ?? ""),
diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.ts
index 1548f5d5..ebebcfff 100644
--- a/apps/web/src/engine/editor.ts
+++ b/apps/web/src/engine/editor.ts
@@ -16,9 +16,10 @@ import { createRoot } from 'solid-js';
import { authoredElement, authoredTree, getRuntimeDocument, insert, isSceneNode, renderAuthored, withDocument } from '@diffusionstudio/reconciler';
+import type { HostNode } from '@diffusionstudio/runtime';
import type { PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';
import type { Entity, World } from 'koota';
-import type { AuthoredTree, HostNode, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';
+import type { AuthoredTree, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';
/**
* A value an edit can carry to the file: what a source spells as a literal,
@@ -794,7 +795,7 @@ export class DocumentEditor {
stage: document.stage,
createElement: (tag) => {
const node = document.createElement(tag);
- if (node !== document.stage) {
+ if (isSceneNode(node) && node !== document.stage) {
document.setProperty(node, SOURCE_ATTR, nextPendingSource());
created.set(node.entity, { tag: tag.charAt(0).toLowerCase() + tag.slice(1), props: {} });
}
diff --git a/packages/encoder/package.json b/packages/encoder/package.json
index 319639ac..e8b797fa 100644
--- a/packages/encoder/package.json
+++ b/packages/encoder/package.json
@@ -1,7 +1,7 @@
{
"name": "@diffusionstudio/encoder",
"version": "0.1.0",
- "description": "Offline video/audio/image encoding over @diffusionstudio/runtime worlds via mediabunny. Needs a browser-grade context (OffscreenCanvas, OfflineAudioContext, AudioWorklet, SharedArrayBuffer).",
+ "description": "Offline video/audio/image encoding over @diffusionstudio/runtime worlds via mediabunny. Needs a browser-grade context (HTMLCanvasElement, OfflineAudioContext, AudioWorklet, SharedArrayBuffer).",
"license": "MPL-2.0",
"type": "module",
"sideEffects": false,
diff --git a/packages/encoder/src/encoder.ts b/packages/encoder/src/encoder.ts
index ef18a665..b1a94294 100644
--- a/packages/encoder/src/encoder.ts
+++ b/packages/encoder/src/encoder.ts
@@ -79,8 +79,12 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {
const sceneSubtree = getEntityTree(sourceWorld, sourceScene);
const subtreeRecords = sceneSubtree.map(entity => serializeEntity(entity));
- // Offscreen canvas
- const offscreenCanvas = new OffscreenCanvas(width, height);
+ // HTML paints need their layout roots attached to the destination canvas.
+ const offscreenCanvas = document.createElement('canvas');
+ offscreenCanvas.width = width;
+ offscreenCanvas.height = height;
+ offscreenCanvas.style.cssText = 'position:fixed;left:-100000px;top:0;pointer-events:none;';
+ document.body.appendChild(offscreenCanvas);
const offscreenCtx = offscreenCanvas.getContext('2d')!;
// Offline audio context
@@ -342,9 +346,9 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {
};
} finally {
URL.revokeObjectURL(audioWorkletUrl);
- // Free the graphs + hosts realized above (this offline world is
- // discarded, so nothing else disposes them — an HtmlHost otherwise
- // leaks a canvas on document.body).
+ offscreenCanvas.remove();
+ // Free the graphs and DOM roots realized above before discarding the
+ // offline world.
mounts?.disposeAll();
// Release decoder file handles/caches, then the world slot itself
// (koota caps live worlds at 16, so throwaway offline worlds must
diff --git a/packages/encoder/src/image-encoder.ts b/packages/encoder/src/image-encoder.ts
index e42ce535..fb20262d 100644
--- a/packages/encoder/src/image-encoder.ts
+++ b/packages/encoder/src/image-encoder.ts
@@ -55,7 +55,11 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncode
rootRecord.End -= rootStart;
}
- const offscreenCanvas = new OffscreenCanvas(2, 2);
+ const offscreenCanvas = document.createElement('canvas');
+ offscreenCanvas.width = 2;
+ offscreenCanvas.height = 2;
+ offscreenCanvas.style.cssText = 'position:fixed;left:-100000px;top:0;pointer-events:none;';
+ document.body.appendChild(offscreenCanvas);
const offscreenCtx = offscreenCanvas.getContext('2d')!;
// Never started — image capture is video-only; the context just satisfies
@@ -218,9 +222,9 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncode
error: e instanceof Error ? e : new Error('Unknown error'),
};
} finally {
- // Free the graphs + hosts realized above (this offline world is
- // discarded; realized HtmlHosts otherwise leak a canvas on
- // document.body), then release decoders and the world slot itself
+ offscreenCanvas.remove();
+ // Free the graphs and DOM roots realized above, then release decoders
+ // and the world slot itself
// (koota caps live worlds at 16).
mounts?.disposeAll();
disposeDecoders(world, world.get(Root)!);
@@ -236,8 +240,10 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncode
};
}
-async function toBase64Png(canvas: OffscreenCanvas): Promise<string> {
- const blob = await canvas.convertToBlob({ type: 'image/png' });
+async function toBase64Png(canvas: HTMLCanvasElement): Promise<string> {
+ const blob = await new Promise<Blob>((resolve, reject) => {
+ canvas.toBlob((value) => value ? resolve(value) : reject(new Error('Could not encode PNG')), 'image/png');
+ });
const dataUrl = await new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index 095e8a56..b68810a5 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -643,7 +643,7 @@ export type HtmlPaintProps = PaintProps & {
children?: SolidJSX.Element;
};
-/** `<Html>` — a rectangle carrying an `<HtmlPaint>` with the given children. */
+/** `<Html>` — a rectangle whose intrinsic paint draws the given DOM children. */
export type HtmlProps = CommonProps & Pick<HtmlPaintProps, "children">;
// HTMLCanvasElement without requiring the DOM lib (this package also
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 2b112c69..8b8646c7 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,28 +3,15 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
+import { SVGElements } from 'solid-js/web';
-import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';
+import type { CameraMatrix, PropertyPath, SceneNode } from '@diffusionstudio/runtime';
import type { AnimatableProperty, AssetRef } from '@diffusionstudio/jsx';
-import { trait, type Entity, type World } from 'koota';
+import type { Entity, World } from 'koota';
import type { ProjectDocument } from './host';
-/**
- * One element of the document, and its place in it. `parent` and `children`
- * are the tree the reconciler reads back: text nodes and element nodes in the
- * one order they were inserted in, which is what the document answers
- * `getFirstChild` and `getNextSibling` from.
- */
-export interface SceneNode {
- readonly entity: Entity;
- tag: string;
- props: Record<string, unknown>;
- parent: SceneNode | null;
- children: HostNode[];
-}
-
/**
* The node a `<surface>` or `<surfacePaint>` element creates: what its `ref`
* receives, like any other element's ref receives its node. `canvas` is the
@@ -36,15 +23,6 @@ export interface SurfaceNode extends SceneNode {
readonly canvas: HTMLCanvasElement | null;
}
-export interface TextNode {
- text: string;
- parent: SceneNode | null;
-}
-
-export type HostNode = SceneNode | TextNode;
-
-const Host = trait(() => null as SceneNode | null);
-
/** Props that address or wire an element rather than describe it. */
const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);
@@ -71,7 +49,7 @@ export interface AuthoredElement {
*/
export function authoredElement(entity: Entity): AuthoredElement | undefined {
const node = entity.get(Host);
- if (!node || node.tag === '') return undefined;
+ if (!node || !node.native || node.tag === '') return undefined;
const text = isText(entity) ? entity.get(Chars)?.value : undefined;
return { tag: node.tag, props: { ...node.props }, ...(text ? { text } : {}) };
@@ -101,15 +79,19 @@ export function authoredTree(world: World, entity: Entity): AuthoredTree | undef
return { ...element, children };
}
-export function isSceneNode(node: HostNode): node is SceneNode {
- return 'entity' in node;
+export function isSceneNode(node: SceneNode | undefined): node is SceneNode {
+ return typeof node === 'object' && node !== null && 'entity' in node;
+}
+
+function isDomText(node: SceneNode | undefined): node is SceneNode & { readonly domNode: Text } {
+ return typeof Text !== 'undefined' && node?.domNode instanceof Text;
}
/**
* The text nodes among `node`'s children, in order; empty when it holds none.
*/
-function textParts(node: SceneNode): TextNode[] {
- return node.children.filter((child): child is TextNode => !isSceneNode(child));
+function textParts(node: SceneNode): (SceneNode & { readonly domNode: Text })[] {
+ return node.children.filter(isDomText);
}
/**
@@ -121,14 +103,14 @@ function syncChars(node: SceneNode): void {
const { entity } = node;
if (!entity.isAlive() || !isText(entity)) return;
entity.add(Chars);
- entity.set(Chars, { value: textParts(node).map((part) => part.text).join('') });
+ entity.set(Chars, { value: textParts(node).map((part) => part.domNode.data).join('') });
}
/**
* Takes `node` out of the children of whatever parent it has, and refreshes
* that parent's text.
*/
-function detach(node: HostNode): void {
+function detach(node: SceneNode): void {
const parent = node.parent;
if (parent === null) return;
@@ -138,19 +120,6 @@ function detach(node: HostNode): void {
syncChars(parent);
}
-/**
- * Cuts the links inside a subtree about to be destroyed, so whatever the
- * reconciler still holds out of it reads as detached — the way its entities
- * read as destroyed.
- */
-function detachSubtree(node: SceneNode): void {
- for (const child of node.children) {
- if (isSceneNode(child)) detachSubtree(child);
- child.parent = null;
- }
- node.children.length = 0;
-}
-
const TIME_TRAITS = {
start: Start,
end: End,
@@ -164,6 +133,7 @@ const PAINT_TYPES: Record<string, PaintType> = {
radialGradientPaint: PaintType.RADIAL_GRADIENT,
shaderPaint: PaintType.SHADER,
surfacePaint: PaintType.SURFACE,
+ htmlPaint: PaintType.HTML,
imagePaint: PaintType.IMAGE,
videoPaint: PaintType.VIDEO,
};
@@ -365,29 +335,6 @@ const FONT_WEIGHTS: Record<string, string> = {
bold: '700',
};
-/**
- * Allocates a surface paint's host with the element (a canvas needs a DOM;
- * a host without one leaves the paint blank and `canvas` null).
- */
-function attachSurfaceHost(paint: Entity): void {
- if (typeof document === 'undefined') return;
- paint.add(SurfaceHostHandle);
- paint.set(SurfaceHostHandle, new SurfaceHost());
-}
-
-/**
- * Sizes the surface canvases of `holder` (its own, for a `<surface>`, and
- * those of its `<surfacePaint>` children) to its authored box, so they draw
- * at the box's resolution. Same-size calls are no-ops (SurfaceHost).
- */
-function syncSurfaceSize(world: World, holder: Entity): void {
- const size = holder.get(Size);
- if (size === undefined) return;
- holder.get(SurfaceHostHandle)?.setSize(size.width, size.height);
- for (const child of getEntityChildren(world, holder)) {
- child.get(SurfaceHostHandle)?.setSize(size.width, size.height);
- }
-}
/**
* A numeric prop's value, or undefined for none. A boolean is none, not 0
@@ -411,14 +358,22 @@ function toSeconds(value: unknown): number | undefined {
return parseTime(value);
}
-export class RuntimeDocument implements ProjectDocument<HostNode> {
+export class RuntimeDocument implements ProjectDocument<SceneNode> {
public readonly stage: SceneNode;
private readonly world: World;
public constructor(world: World) {
this.world = world;
const root = world.get(Root)!;
- this.stage = { entity: root, tag: '', props: {}, parent: null, children: [] };
+ this.stage = {
+ entity: root,
+ native: true,
+ tag: '',
+ props: {},
+ parent: null,
+ children: [],
+ domNode: null,
+ };
root.add(Host);
root.set(Host, this.stage);
}
@@ -434,18 +389,55 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
const held = entity.get(Host);
if (held) return held;
- const node: SceneNode = { entity, tag: '', props: {}, parent: null, children: [] };
+ const node: SceneNode = {
+ entity,
+ native: true,
+ tag: '',
+ props: {},
+ parent: null,
+ children: [],
+ domNode: null,
+ };
entity.add(Host);
entity.set(Host, node);
return node;
}
- public createElement(tag: string): SceneNode | SurfaceNode {
- // Composition elements arrive in either spelling: the camelCase
- // intrinsics a project authors, or the PascalCase components the compile
- // step (and an app inserting elements) uses.
+ public createElement(tag: string): SceneNode {
+ // camelCase composition tags are DOM elements.
+ if (tag.charAt(0) === tag.charAt(0).toLowerCase()) {
+ if (typeof document === 'undefined') {
+ throw new Error(`<${tag}> requires a DOM`);
+ }
+ if (tag === 'audio' || tag === 'video') {
+ throw new Error(
+ `<${tag}> is not supported as HTML content; use the <${tag}> composition element outside the HTML subtree`,
+ );
+ }
+ if (tag === 'canvas') {
+ throw new Error('<canvas> is not supported as HTML content; use <surface> for custom drawing');
+ }
+ const domNode = SVGElements.has(tag)
+ ? document.createElementNS('http://www.w3.org/2000/svg', tag)
+ : document.createElement(tag);
+ const entity = createEntity(this.world);
+ const node: SceneNode = {
+ entity,
+ native: false,
+ tag,
+ props: {},
+ parent: null,
+ children: [],
+ domNode,
+ };
+ entity.add(Host);
+ entity.set(Host, node);
+ return node;
+ }
+
const name = tag.charAt(0).toLowerCase() + tag.slice(1);
let entity: Entity;
+ let domNode: Element | null = null;
switch (name) {
case 'stage':
@@ -561,6 +553,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
case 'radialGradientPaint':
case 'shaderPaint':
case 'surfacePaint':
+ case 'htmlPaint':
case 'imagePaint':
case 'videoPaint': {
entity = createEntity(this.world);
@@ -568,7 +561,42 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
entity.set(Paint, { value: PAINT_TYPES[name]! });
if (name === 'solidPaint') entity.add(Color);
if (name === 'shaderPaint') entity.add(Shader);
- if (name === 'surfacePaint') attachSurfaceHost(entity);
+ if (name === 'surfacePaint') {
+ domNode = document.createElement('canvas');
+ }
+ if (name === 'htmlPaint') {
+ const root = document.createElement('div');
+ root.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';
+
+ const canvas = this.world.get(RenderSurface)?.canvas;
+ assert(canvas instanceof HTMLCanvasElement, 'HTML paint must be on a canvas');
+
+ canvas.toggleAttribute('layoutsubtree', true);
+ canvas.appendChild(root);
+
+ domNode = root;
+ }
+ break;
+ }
+ case 'html': {
+ entity = createEntity(this.world);
+ entity.add(Geometry);
+ entity.set(Geometry, { value: GeometryType.RECT });
+ entity.add(Position);
+ entity.set(Position, { x: 0, y: 0 });
+ entity.add(Paint);
+ entity.set(Paint, { value: PaintType.HTML });
+
+ const root = document.createElement('div');
+ root.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';
+ const canvas = this.world.get(RenderSurface)?.canvas;
+ assert(canvas instanceof HTMLCanvasElement, 'HTML paint must be on a canvas');
+
+ canvas.toggleAttribute('layoutsubtree', true);
+ canvas.appendChild(root);
+
+ domNode = root;
+ resizeEntity(this.world, entity, { width: 100, height: 100 });
break;
}
case 'surface': {
@@ -581,9 +609,8 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
entity.set(Position, { x: 0, y: 0 });
entity.add(Paint);
entity.set(Paint, { value: PaintType.SURFACE });
- attachSurfaceHost(entity);
+ domNode = document.createElement('canvas');
resizeEntity(this.world, entity, { width: 100, height: 100 });
- syncSurfaceSize(this.world, entity);
break;
}
case 'colorStop': {
@@ -633,7 +660,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
}
default:
throw new Error(
- `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <captions>, <adjustmentLayer>, <rect>, <text>, <textRange>, <video>, <image>, <audio>, <surface>, <solidPaint>, <linearGradientPaint>, <radialGradientPaint>, <shaderPaint>, <surfacePaint>, <imagePaint>, <videoPaint>, <colorStop>, <stroke>, <shadow>, <effect>, <animation>, <keyframeTrack> and <keyframe>).`,
+ `<${tag}> is not supported yet (only composition elements exported by @diffusionstudio/jsx are accepted).`,
);
}
@@ -645,18 +672,20 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
const node: SceneNode = {
entity,
+ native: true,
tag: name,
props: {},
parent: null,
- children: []
+ children: [],
+ domNode,
};
if (name === 'surface' || name === 'surfacePaint') {
Object.assign(node, {
get canvas() {
- return entity.get(SurfaceHostHandle)?.canvas ?? null;
+ return node.domNode instanceof HTMLCanvasElement ? node.domNode : null;
},
- })
+ });
}
entity.add(Host);
@@ -664,20 +693,31 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return node;
}
- public createTextNode(text: string): TextNode {
- return { text, parent: null };
+ public createTextNode(text: string): SceneNode {
+ const entity = createEntity(this.world);
+ const node: SceneNode = {
+ entity,
+ native: false,
+ tag: '#text',
+ props: {},
+ parent: null,
+ children: [],
+ domNode: document.createTextNode(text),
+ };
+ entity.add(Host);
+ entity.set(Host, node);
+ return node;
}
- public replaceText(node: HostNode, text: string): void {
- if (isSceneNode(node)) return;
- node.text = text;
- if (node.parent) {
- syncChars(node.parent);
- }
+ public replaceText(node: SceneNode, text: string): void {
+ if (!isDomText(node)) return;
+ node.domNode.data = text;
+ const owner = node.parent;
+ if (owner) syncChars(owner);
}
- public isTextNode(node: HostNode): boolean {
- return !isSceneNode(node);
+ public isTextNode(node: SceneNode): boolean {
+ return isDomText(node);
}
/**
@@ -699,13 +739,18 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
}
for (const [index, part] of parts.entries()) {
- part.text = index === 0 ? text : '';
+ part.domNode.data = index === 0 ? text : '';
}
syncChars(node);
}
- public setProperty(node: HostNode, name: string, value: unknown): void {
+ public setProperty(node: SceneNode, name: string, value: unknown): void {
if (!isSceneNode(node)) return;
+
+ if (!node.native && node.domNode instanceof Element) {
+ setDomProperty(node.domNode, name, value);
+ }
+
const { entity } = node;
// The stage and anything adopted from outside are no element of the
@@ -932,7 +977,6 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return;
}
resizeEntity(this.world, entity, { [name]: size });
- syncSurfaceSize(this.world, entity);
return;
}
case 'join': {
@@ -1368,21 +1412,64 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
* next value takes its place. Only a `<text>` reads its text children back
* out as `Chars`.
*/
- public insertNode(parent: HostNode, node: HostNode, anchor?: HostNode): void {
- if (!isSceneNode(parent)) {
+ public insertNode(parent: SceneNode, node: SceneNode, anchor?: SceneNode): void {
+ if (isDomText(parent)) {
throw new Error('Text cannot contain children.');
}
- if (isSceneNode(node)) {
- if (parent.entity === node.entity) return;
- // Checked before anything moves: `appendChild` asserts the same
- // thing, but only once the node has left the parent it had.
- if (
- getParentEntity(node.entity) !== parent.entity &&
- getEntityTree(this.world, node.entity).includes(parent.entity)
- ) {
- throw new Error('Cannot parent entity into its own subtree');
+ const candidate = (!parent.native || parent.tag === 'html' || parent.tag === 'htmlPaint')
+ ? parent.domNode
+ : null;
+ const domContainer = candidate instanceof Element ? candidate : null;
+
+ if (isDomText(node)) {
+ detach(node);
+ const at = anchor === undefined ? -1 : parent.children.indexOf(anchor);
+ if (at === -1) parent.children.push(node);
+ else parent.children.splice(at, 0, node);
+ node.parent = parent;
+
+ if (domContainer) {
+ const domAnchor = anchor?.domNode;
+ domContainer.insertBefore(
+ node.domNode,
+ domAnchor?.parentNode === domContainer ? domAnchor : null,
+ );
+ } else {
+ node.domNode.remove();
+ }
+ syncChars(parent);
+ return;
+ }
+
+ if (domContainer) {
+ if (node.native || !(node.domNode instanceof Element)) {
+ throw new Error('A composition element cannot be HTML content.');
}
+
+ detach(node);
+ const at = anchor === undefined ? -1 : parent.children.indexOf(anchor);
+ if (at === -1) parent.children.push(node);
+ else parent.children.splice(at, 0, node);
+ node.parent = parent;
+
+ const domAnchor = anchor?.domNode;
+ domContainer.insertBefore(node.domNode, domAnchor?.parentNode === domContainer ? domAnchor : null);
+ return;
+ }
+
+ if (!node.native) {
+ throw new Error('DOM content must be inside <html> or <htmlPaint>.');
+ }
+
+ if (parent.entity === node.entity) return;
+ // Checked before anything moves: `appendChild` asserts the same
+ // thing, but only once the node has left the parent it had.
+ if (
+ getParentEntity(node.entity) !== parent.entity &&
+ getEntityTree(this.world, node.entity).includes(parent.entity)
+ ) {
+ throw new Error('Cannot parent entity into its own subtree');
}
detach(node);
@@ -1391,11 +1478,6 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
else parent.children.splice(at, 0, node);
node.parent = parent;
- if (!isSceneNode(node)) {
- syncChars(parent);
- return;
- }
-
if (getParentEntity(node.entity) !== parent.entity) {
// appendChild only takes top-level entities, so a move between two
// parents goes back through the stage on the way.
@@ -1410,29 +1492,37 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
if (node.entity.has(KeyframeTrack)) {
this.resolveTrackProperty(node.entity, node.props.property);
}
- if (node.entity.has(SurfaceHostHandle)) {
- syncSurfaceSize(this.world, parent.entity);
- }
}
let index = 0;
for (const sibling of parent.children) {
- if (!isSceneNode(sibling)) continue;
+ if (!sibling.native) continue;
sibling.entity.add(ItemIndex);
sibling.entity.set(ItemIndex, { value: index++ });
}
}
+ private destroySubtree(node: SceneNode): void {
+ for (const child of [...node.children]) {
+ this.destroySubtree(child);
+ }
+ node.children.length = 0;
+ node.parent = null;
+ node.domNode?.remove();
+
+ if (node.entity.isAlive() && node.entity !== this.stage.entity) {
+ node.entity.destroy();
+ }
+ }
+
/**
* Takes `node` out of the document. The parent it comes out of is the one
* it has rather than the one the caller names: a move has already put it
* in the new one by the time the renderer asks the old one to drop it.
*/
- public removeNode(_parent: HostNode, node: HostNode): void {
+ public removeNode(_parent: SceneNode, node: SceneNode): void {
detach(node);
- if (!isSceneNode(node)) return;
-
if (node.entity.has(Stage)) {
for (const child of [...node.children]) {
this.removeNode(node, child);
@@ -1442,23 +1532,18 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return;
}
- // Destroy cascades through the subtree; whatever the reconciler still
- // holds out of it — a text node, a node a ref kept — learns it has no
- // parent, the way the entities learn they are gone.
- detachSubtree(node);
- // ChildOf auto-destroys orphans, so paints go with it.
- node.entity.destroy();
+ this.destroySubtree(node);
}
- public getParentNode(node: HostNode): SceneNode | undefined {
+ public getParentNode(node: SceneNode): SceneNode | undefined {
return node.parent ?? undefined;
}
- public getFirstChild(node: HostNode): HostNode | undefined {
- return isSceneNode(node) ? node.children[0] : undefined;
+ public getFirstChild(node: SceneNode): SceneNode | undefined {
+ return node.children[0];
}
- public getNextSibling(node: HostNode): HostNode | undefined {
+ public getNextSibling(node: SceneNode): SceneNode | undefined {
const parent = node.parent;
if (parent === null) return undefined;
const at = parent.children.indexOf(node);
@@ -1467,7 +1552,18 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
public dispose(): void {
- this.removeNode(this.stage, this.stage);
+ for (const entity of this.world.entities) {
+ if (!entity.isAlive() || entity === this.stage.entity) continue;
+ const domNode = entity.get(Host)?.domNode;
+ if (domNode instanceof HTMLImageElement && domNode.src.startsWith('blob:')) {
+ URL.revokeObjectURL(domNode.src);
+ }
+ if (domNode) {
+ domNode.remove();
+ }
+ entity.destroy();
+ }
+
// The world is free for the next mount. A document already replaced by
// another leaves the registry as it found it.
if (documents.get(this.world) === this) {
@@ -1476,6 +1572,65 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
}
}
+/**
+ * Prop assignment for real DOM nodes below an HTML paint. This mirrors the
+ * useful, non-interactive part of Solid's DOM conventions: reactive styles,
+ * classes, attributes and explicit HTML/text content. Pointer and media event
+ * handlers are ignored because the subtree is rasterized into the canvas.
+ */
+function setDomProperty(element: Element | null, name: string, value: unknown): void {
+ if (
+ typeof value === 'function'
+ || name.startsWith('on')
+ || element === null
+ || name === 'children'
+ || name === 'ref'
+ ) return;
+
+ if (name === 'style') {
+ const style = (element as HTMLElement | SVGElement).style;
+ if (typeof value === 'string') {
+ style.cssText = value;
+ } else if (typeof value === 'object' && value !== null) {
+ Object.assign(style, value);
+ } else {
+ style.cssText = '';
+ }
+ return;
+ }
+
+ if (name === 'class' || name === 'className') {
+ if (value === undefined || value === null || value === false) element.removeAttribute('class');
+ else element.setAttribute('class', String(value));
+ return;
+ }
+
+ if (name === 'classList') {
+ if (typeof value === 'object' && value !== null) {
+ for (const [key, enabled] of Object.entries(value)) {
+ element.classList.toggle(key, Boolean(enabled));
+ }
+ }
+ return;
+ }
+
+ if (name === 'innerHTML') {
+ element.innerHTML = value === undefined || value === null ? '' : String(value);
+ return;
+ }
+
+ if (name === 'textContent') {
+ element.textContent = value === undefined || value === null ? '' : String(value);
+ return;
+ }
+
+ if (value === undefined || value === null || value === false) {
+ element.removeAttribute(name);
+ } else {
+ element.setAttribute(name, value === true ? '' : String(value));
+ }
+}
+
const documents = new WeakMap<World, RuntimeDocument>();
export function getRuntimeDocument(world: World): RuntimeDocument {
@@ -1504,3 +1659,9 @@ export function createRuntimeDocument(world: World): RuntimeDocument {
documents.set(world, document);
return document;
}
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
diff --git a/packages/reconciler/src/elements.ts b/packages/reconciler/src/elements.ts
index 9a1eb002..64c59458 100644
--- a/packages/reconciler/src/elements.ts
+++ b/packages/reconciler/src/elements.ts
@@ -39,6 +39,7 @@ import type {
ShaderPaintProps,
ShadowProps,
SolidPaintProps,
+ StageProps,
StrokeProps,
SurfacePaintProps,
SurfaceProps,
@@ -71,7 +72,7 @@ function hostElement<P extends object>(tag: string): (props: P) => SolidJSX.Elem
* editor renders it inside `insertElement`'s thunk to duplicate a subtree.
*/
export function renderAuthored(tree: AuthoredTree): unknown {
- const node = createElement(tree.tag);
+ const node = createElement(tree.tag.charAt(0).toUpperCase() + tree.tag.slice(1));
for (const [name, value] of Object.entries(tree.props)) {
setProp(node, name, value);
}
@@ -84,6 +85,7 @@ export function renderAuthored(tree: AuthoredTree): unknown {
return node;
}
+export const Stage = hostElement<StageProps>("Stage");
export const Scene = hostElement<SceneProps>("Scene");
export const Group = hostElement<GroupProps>("Group");
export const Rect = hostElement<RectProps>("Rect");
diff --git a/packages/runtime/src/actions/resize.ts b/packages/runtime/src/actions/resize.ts
index c81dc327..3d48e545 100644
--- a/packages/runtime/src/actions/resize.ts
+++ b/packages/runtime/src/actions/resize.ts
@@ -9,6 +9,7 @@ import { ConstraintType } from '../constants';
import {
ChildOf, Group, Size, Position, Constraint, ConstraintCache,
KeepAspectRatio, Sequential, Computed,
+ Host,
} from '../traits';
import { getParentNode } from '../queries/hierarchy';
@@ -57,6 +58,20 @@ export function resizeEntity(world: World, entity: Entity, params: ResizeParams)
...(width !== undefined && { width: Math.round(width) }),
...(height !== undefined && { height: Math.round(height) }),
});
+
+ // handle DOM nodes
+ const domNode = entity.get(Host)?.domNode;
+
+ if (domNode instanceof HTMLCanvasElement) {
+ domNode.width = width ?? domNode.width;
+ domNode.height = height ?? domNode.height;
+ }
+
+ if (domNode instanceof HTMLElement) {
+ domNode.style.width = `${width ?? domNode.style.width.replace('px', '')}px`;
+ domNode.style.height = `${height ?? domNode.style.height.replace('px', '')}px`;
+ }
+
// The Size observer propagates Computed sizes; constraints resolve here.
resolveConstraintOffsets(world, entity);
}
diff --git a/packages/runtime/src/media/html.ts b/packages/runtime/src/media/html.ts
index ec4de380..0738fd8d 100644
--- a/packages/runtime/src/media/html.ts
+++ b/packages/runtime/src/media/html.ts
@@ -2,15 +2,6 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
-
-// html-in-canvas (https://github.com/WICG/html-in-canvas). Chromium only,
-// behind chrome://flags/#canvas-draw-element; the API surface is still
-// moving, so every touchpoint is typed and isolated here.
-type DrawElementContext = Ctx2D & {
- drawElementImage(source: Element | unknown, dx: number, dy: number, dw: number, dh: number): DOMMatrix;
-};
-
export function isHtmlInCanvasSupported(): boolean {
return typeof CanvasRenderingContext2D !== 'undefined'
&& 'drawElementImage' in CanvasRenderingContext2D.prototype;
@@ -31,122 +22,18 @@ export function nextRenderingUpdate(): Promise<void> {
});
}
-let htmlCanvas: HTMLCanvasElement | null = null;
-let htmlCtx: CanvasRenderingContext2D | null = null;
-let htmlHosts = 0;
-
-function acquireLayoutCanvas(): HTMLCanvasElement {
- if (!htmlCanvas) {
- htmlCanvas = document.createElement('canvas');
- htmlCanvas.width = 1;
- htmlCanvas.height = 1;
- htmlCanvas.toggleAttribute('layoutsubtree', true);
- htmlCanvas.style.cssText = 'position:fixed;left:0;top:0;width:1px;height:1px;pointer-events:none;';
- document.body.appendChild(htmlCanvas);
- htmlCtx = htmlCanvas.getContext('2d')!;
- }
- htmlHosts++;
- return htmlCanvas;
-}
-
-function releaseLayoutCanvas(): void {
- if (--htmlHosts === 0) {
- htmlCanvas?.remove();
- htmlCanvas = null;
- htmlCtx = null;
- }
-}
-
-/**
- * DOM host for one HTML paint: a div inside the shared layoutsubtree canvas.
- * The mount document inserts the paint's JSX children into `element` as real,
- * reactive DOM; the browser lays them out at the parent geometry's box size
- * and `draw` rasterizes the result at the destination's device scale and
- * blits it into the rendering context (stage or offscreen alike).
- */
-export class HtmlHost {
- public readonly element: HTMLDivElement;
- private disposed = false;
- private warned = false;
- private width = -1;
- private height = -1;
-
- public constructor() {
- const el = document.createElement('div');
- // pointer-events: none — layoutsubtree children participate in hit
- // testing and would otherwise swallow page interactions.
- el.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';
- acquireLayoutCanvas().appendChild(el);
- this.element = el;
+/** Resolve asynchronously loaded resources below an HTML paint root. */
+export function whenHtmlReady(root: HTMLElement, timeInSeconds: number): Promise<void> {
+ for (const animation of root.getAnimations({ subtree: true })) {
+ if (animation.playState !== 'paused') animation.pause();
+ animation.currentTime = timeInSeconds * 1000;
}
- /**
- * Resolves asynchronously loaded html resources.
- */
- public whenReady(timeInSeconds: number): Promise<void> {
- // sync animations
- for (const animation of this.element.getAnimations({ subtree: true })) {
- if (animation.playState !== 'paused') animation.pause();
- animation.currentTime = timeInSeconds * 1000;
- }
-
- // wait for fonts to be ready
- const pending: Promise<unknown>[] = [document.fonts.ready];
-
- // wait for images to be ready
- for (const image of this.element.querySelectorAll('img')) {
- // `complete` covers loaded, failed, and srcless images alike; the
- // rest are still in flight, and a failed decode just paints nothing.
- if (image.complete) continue;
- pending.push(image.decode().catch(() => undefined));
- }
-
- return Promise.all(pending).then(() => nextRenderingUpdate());
- }
-
- public setSize(width: number, height: number): void {
- if (width === this.width && height === this.height) return;
- this.width = width;
- this.height = height;
- this.element.style.width = `${width}px`;
- this.element.style.height = `${height}px`;
- }
-
- public draw(ctx: Ctx2D, width: number, height: number): void {
- if (this.disposed || !htmlCanvas || !htmlCtx) return;
-
- this.setSize(width, height);
-
- try {
- const m = ctx.getTransform();
- const sx = Math.hypot(m.a, m.b) || 1;
- const sy = Math.hypot(m.c, m.d) || 1;
- const pw = Math.max(1, Math.ceil(width * sx));
- const ph = Math.max(1, Math.ceil(height * sy));
-
- // Grow-only: shrinking would clear and reallocate every time hosts
- // of different sizes share the canvas within one frame.
- if (htmlCanvas.width < pw) htmlCanvas.width = pw;
- if (htmlCanvas.height < ph) htmlCanvas.height = ph;
-
- htmlCtx.setTransform(sx, 0, 0, sy, 0, 0);
- htmlCtx.clearRect(0, 0, width, height);
- (htmlCtx as DrawElementContext).drawElementImage(this.element, 0, 0, width, height);
- ctx.drawImage(htmlCanvas, 0, 0, pw, ph, 0, 0, width, height);
- } catch (e) {
- // A transient failure (subtree not yet laid out or painted) just
- // skips the frame; log once so a permanent one doesn't spam.
- if (!this.warned) {
- this.warned = true;
- console.error(`Error drawing <HtmlPaint> content: ${e instanceof Error ? `${e.name}: ${e.message}` : e}`);
- }
- }
+ const pending: Promise<unknown>[] = [document.fonts.ready];
+ for (const image of root.querySelectorAll('img')) {
+ if (image.complete) continue;
+ pending.push(image.decode().catch(() => undefined));
}
- public dispose(): void {
- if (this.disposed) return;
- this.disposed = true;
- this.element.remove();
- releaseLayoutCanvas();
- }
+ return Promise.all(pending).then(() => nextRenderingUpdate());
}
diff --git a/packages/runtime/src/media/index.ts b/packages/runtime/src/media/index.ts
index 61a4d715..b30f5a43 100644
--- a/packages/runtime/src/media/index.ts
+++ b/packages/runtime/src/media/index.ts
@@ -15,7 +15,6 @@ export * from './image';
export * from './keyframe-index';
export * from './sequence';
export * from './shader';
-export * from './surface';
export * from './time-stretcher';
export * from './transcode';
export * from './video';
diff --git a/packages/runtime/src/media/surface.ts b/packages/runtime/src/media/surface.ts
deleted file mode 100644
index bd31a790..00000000
--- a/packages/runtime/src/media/surface.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
-
-/**
- * Backing store for one surface paint: a detached canvas handed to the
- * mount's `ref` callback, which draws into it with any context type (2d,
- * webgl, webgpu). The engine samples the bitmap every frame and stretches it
- * into the parent geometry's box. The host allocates the bitmap at the box
- * size and follows the box's authored size; user code (e.g. three.js via
- * `renderer.setSize`) may resize it through its own APIs as well.
- */
-export class SurfaceHost {
- public readonly canvas = document.createElement('canvas');
- private disposed = false;
-
- /**
- * Bitmap allocation at the box size. A no-op when the size is unchanged:
- * assigning a canvas dimension clears the bitmap even when the value is
- * the same, and user code may already have applied this size itself.
- */
- public setSize(width: number, height: number): void {
- const w = Math.max(1, Math.round(width));
- const h = Math.max(1, Math.round(height));
- if (this.canvas.width !== w) this.canvas.width = w;
- if (this.canvas.height !== h) this.canvas.height = h;
- }
-
- public draw(ctx: Ctx2D, width: number, height: number): void {
- if (this.disposed || this.canvas.width === 0 || this.canvas.height === 0) return;
- ctx.drawImage(this.canvas, 0, 0, width, height);
- }
-
- public dispose(): void {
- if (this.disposed) return;
- this.disposed = true;
- // Release the backing store; context teardown (e.g. a WebGL renderer)
- // is the mount's own onCleanup concern.
- this.canvas.width = 0;
- this.canvas.height = 0;
- }
-}
diff --git a/packages/runtime/src/queries/predicates.ts b/packages/runtime/src/queries/predicates.ts
index e54860ef..a7703300 100644
--- a/packages/runtime/src/queries/predicates.ts
+++ b/packages/runtime/src/queries/predicates.ts
@@ -64,9 +64,11 @@ export function isMask(entity: Entity): boolean {
}
export function hasHtmlPaint(entity: Entity): boolean {
- return (entity.get(Cache)?.fills ?? []).some(fill => fill.get(Paint)?.value === PaintType.HTML);
+ return entity.get(Paint)?.value === PaintType.HTML
+ || (entity.get(Cache)?.fills ?? []).some(fill => fill.get(Paint)?.value === PaintType.HTML);
}
export function hasSurfacePaint(entity: Entity): boolean {
- return (entity.get(Cache)?.fills ?? []).some(fill => fill.get(Paint)?.value === PaintType.SURFACE);
+ return entity.get(Paint)?.value === PaintType.SURFACE
+ || (entity.get(Cache)?.fills ?? []).some(fill => fill.get(Paint)?.value === PaintType.SURFACE);
}
diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.ts
index 675a7918..cf88620e 100644
--- a/packages/runtime/src/systems/assets.ts
+++ b/packages/runtime/src/systems/assets.ts
@@ -2,14 +2,23 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Ai, FramePromises, Generating, GenerationRequest, Library, LoadRequest, PendingSource, SourceError, TranscriptionRequest } from '../traits';
-import { bindAsset, getModifiers } from '../actions/assets';
+import { Ai, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, SourceError, TranscriptionRequest } from '../traits';
+import { bindAsset, getAssetFile, getModifiers } from '../actions/assets';
import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';
import type { Entity, World } from 'koota';
import type { Asset } from '@diffusionstudio/assets';
export function assetSystem(world: World): void {
+ for (const entity of world.query(LoadRequest, Host)) {
+ const source = entity.get(LoadRequest)!.value;
+ if (!isDomImage(entity)) continue;
+ pointDomImageAt(entity, null);
+ if (source !== '' && !/^(?:data|blob):/i.test(source)) continue;
+ entity.remove(LoadRequest);
+ pointDomImageAt(entity, source || null);
+ }
+
const library = world.get(Library);
if (library) {
for (const entity of world.query(LoadRequest)) {
@@ -25,6 +34,7 @@ export function assetSystem(world: World): void {
const ref = entity.get(GenerationRequest)!.ref;
entity.remove(GenerationRequest);
if (ref === null || entity.has(SourceError)) continue;
+ if (isDomImage(entity)) pointDomImageAt(entity, null);
start(world, entity, ai.resolve(ref), true);
}
@@ -88,24 +98,59 @@ function resolve(world: World, entity: Entity, promise: Promise<Asset>, generati
entity.set(PendingSource, { value: token });
if (generating) entity.add(Generating);
- const done = promise.then(
- (asset) => {
- if (!current(entity, token)) return;
- entity.remove(PendingSource, Generating);
- bindAsset(entity, asset);
- },
- (error: unknown) => {
- if (!current(entity, token)) return;
- entity.remove(PendingSource, Generating);
- entity.add(SourceError);
- entity.set(SourceError, { value: errorMessage(error), generated: generating });
- console.error('[runtime] could not resolve src:', error);
- },
- );
+ const done = promise.then(async (asset) => {
+ if (!current(entity, token)) return;
+ bindAsset(entity, asset);
+ await bindDomImage(entity, asset, token);
+ if (current(entity, token)) entity.remove(PendingSource, Generating);
+ }).catch((error: unknown) => {
+ if (!current(entity, token)) return;
+ entity.remove(PendingSource, Generating);
+ entity.add(SourceError);
+ entity.set(SourceError, { value: errorMessage(error), generated: generating });
+ console.error('[runtime] could not resolve src:', error);
+ });
world.get(FramePromises)?.list?.push(done);
}
+function isDomImage(entity: Entity): boolean {
+ return typeof HTMLImageElement !== 'undefined'
+ && entity.get(Host)?.domNode instanceof HTMLImageElement;
+}
+
+function pointDomImageAt(entity: Entity, source: string | null): void {
+ const node = entity.get(Host);
+ const image = node?.domNode;
+ if (typeof HTMLImageElement === 'undefined' || !(image instanceof HTMLImageElement) || !node) return;
+
+ releaseDomImageSource(image);
+ if (source === null) image.removeAttribute('src');
+ else image.src = source;
+}
+
+export function releaseDomImageSource(image: HTMLImageElement): void {
+ if (image.src.startsWith('blob:')) URL.revokeObjectURL(image.src);
+}
+
+async function bindDomImage(entity: Entity, asset: Asset, token: object): Promise<void> {
+ if (!isDomImage(entity)) return;
+ const file = await getAssetFile(asset);
+ if (!current(entity, token)) return;
+
+ const url = URL.createObjectURL(file);
+ if (!current(entity, token)) {
+ URL.revokeObjectURL(url);
+ return;
+ }
+
+ pointDomImageAt(entity, url);
+ const image = entity.get(Host)?.domNode;
+ if (typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement) {
+ await image.decode().catch(() => undefined);
+ }
+}
+
/** What a rejection says, for an entity to carry and the host to show. */
function errorMessage(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
diff --git a/packages/runtime/src/systems/playback.ts b/packages/runtime/src/systems/playback.ts
index 3b9f4c15..8ce2699d 100644
--- a/packages/runtime/src/systems/playback.ts
+++ b/packages/runtime/src/systems/playback.ts
@@ -16,7 +16,7 @@ import {
Geometry, Group, AdjustmentLayer, Paint, Audio, Caption, Muted, Soloed,
Sequential, Transition, Playback, Workarea,
AudioPlayback, Computed,
- AudioDecoderHandle, AudioBusHandle, HtmlHostHandle,
+ AudioDecoderHandle, AudioBusHandle, Host,
Mode, FrameRate, Time, AudioEngine, FramePromises, Mounts,
Root,
} from '../traits';
@@ -28,6 +28,7 @@ import {
resolveAudioDecoder, resolveCaptionDecoder, resolveImageDecoder,
resolveShaderHost, resolveVideoDecoder,
} from '../media';
+import { whenHtmlReady } from '../media/html';
import { AudioBus } from '../media/audio-bus';
import type { Entity, World } from 'koota';
@@ -201,9 +202,11 @@ function forwardAudioDecoder(world: World, scene: Entity, entity: Entity, audioS
function forwardHtmlHost(world: World, scene: Entity, entity: Entity, fill: Entity): void {
const computed = store(world, Computed);
- const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;
- if (world.get(Mode)?.value === 'realtime' || computed.visibility[entity.id()] !== 1 || !host) return;
- framePromises(world)?.push(host.whenReady(computed.localTimeInSeconds[scene.id()] ?? 0));
+ const root = fill.get(Host)?.domNode;
+ if (world.get(Mode)?.value === 'realtime'
+ || computed.visibility[entity.id()] !== 1
+ || !(root instanceof HTMLElement)) return;
+ framePromises(world)?.push(whenHtmlReady(root, computed.localTimeInSeconds[scene.id()] ?? 0));
}
function forwardImageDecoder(world: World, _scene: Entity, _entity: Entity, fill: Entity): void {
@@ -238,6 +241,8 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {
intrinsicVideo = true;
} else if (intrinsic === PaintType.IMAGE && visualsEnabled) {
forwardImageDecoder(world, scene, entity, entity);
+ } else if (intrinsic === PaintType.HTML && visualsEnabled) {
+ forwardHtmlHost(world, scene, entity, entity);
}
for (const fill of world.query(ChildOf(entity), Paint, Not(Geometry), Not(Hidden))) {
diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.ts
index ff68a6d1..6230d1d8 100644
--- a/packages/runtime/src/systems/render.ts
+++ b/packages/runtime/src/systems/render.ts
@@ -3,8 +3,8 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// Render system (was systems/render.ts): draws the document tree onto the
-// world's RenderSurface. Runs identically against the editor canvas and an
-// OffscreenCanvas during capture; without a surface it is a no-op. Hit
+// world's RenderSurface. Runs identically against the editor and capture
+// canvases; without a surface it is a no-op. Hit
// regions are pushed callback-less (see HitRegions); the app's input layer
// attaches its handlers.
@@ -21,7 +21,7 @@ import {
SourceError,
BlendMode, Effect, Transition, MixedCornerRadius,
LocalTransform, WorldTransform, Computed, Cache,
- HtmlHostHandle, SurfaceHostHandle,
+ Host,
Mode, FrameRate, Camera, Background, RenderSurface,
HitRegions,
Root,
@@ -215,18 +215,22 @@ export function renderIntrinsicFill(world: World, entity: Entity): void {
const intrinsic = getIntrinsicPaint(entity);
if (intrinsic === PaintType.SURFACE) {
- const host = entity.has(SurfaceHostHandle) ? entity.get(SurfaceHostHandle) : null;
- if (host) {
+ const canvas = entity.get(Host)?.domNode;
+ if (canvas instanceof HTMLCanvasElement) {
const ctx = getCtx(world);
const computed = store(world, Computed);
const eid = entity.id();
ctx.save();
ctx.clip();
- host.draw(ctx, computed.width[eid]!, computed.height[eid]!);
+ ctx.drawImage(canvas, 0, 0, computed.width[eid]!, computed.height[eid]!);
ctx.restore();
}
return;
}
+ if (intrinsic === PaintType.HTML) {
+ renderHtmlFill(world, entity, entity);
+ return;
+ }
const kind = mediaKind(intrinsic);
if (kind === null) return;
@@ -245,6 +249,41 @@ function mediaKind(paint: PaintType | undefined): MediaKind | null {
/** What the image and video decoders hand out to draw. */
type MediaFrame = ImageBitmap | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas;
+// html-in-canvas (https://github.com/WICG/html-in-canvas). Chromium only,
+// behind chrome://flags/#canvas-draw-element; the API surface is still
+// moving, so every touchpoint is typed and isolated here.
+type DrawElementContext = Ctx2D & {
+ drawElementImage(source: Element | unknown, dx: number, dy: number, dw: number, dh: number): DOMMatrix;
+};
+
+
+function renderHtmlFill(world: World, entity: Entity, source: Entity): void {
+ const root = source.get(Host)?.domNode;
+ const surface = world.get(RenderSurface);
+ const ctx = surface?.ctx;
+ if (!(ctx instanceof CanvasRenderingContext2D)) return;
+ if (!(root instanceof HTMLElement)) return;
+
+ const computed = store(world, Computed);
+ const eid = entity.id();
+
+ ctx.save();
+ ctx.clip();
+
+ const width = computed.width[eid]!;
+ const height = computed.height[eid]!;
+ root.style.width = `${width}px`;
+ root.style.height = `${height}px`;
+
+ try {
+ (ctx as DrawElementContext).drawElementImage(root, 0, 0, width, height);
+ } catch (error) {
+ console.error(`Error drawing <HtmlPaint> content: ${error instanceof Error ? `${error.name}: ${error.message}` : error}`);
+ }
+
+ ctx.restore();
+}
+
/**
* Draws the current frame of `source` (an image or video paint, or the
* geometry itself for intrinsic media) into `entity`'s box, fitted by the
@@ -312,21 +351,13 @@ export function renderFills(world: World, entity: Entity): void {
} else if (paint === PaintType.VIDEO) {
renderMedia(world, entity, fill, 'VIDEO');
} else if (paint === PaintType.HTML) {
- const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;
-
- if (host) {
- ctx.save();
- ctx.clip();
- host.draw(ctx, computed.width[eid]!, computed.height[eid]!);
- ctx.restore();
- }
+ renderHtmlFill(world, entity, fill);
} else if (paint === PaintType.SURFACE) {
- const host = fill.has(SurfaceHostHandle) ? fill.get(SurfaceHostHandle) : null;
-
- if (host) {
+ const canvas = fill.get(Host)?.domNode;
+ if (canvas instanceof HTMLCanvasElement) {
ctx.save();
ctx.clip();
- host.draw(ctx, computed.width[eid]!, computed.height[eid]!);
+ ctx.drawImage(canvas, 0, 0, computed.width[eid]!, computed.height[eid]!);
ctx.restore();
}
} else if (paint === PaintType.SOLID) {
diff --git a/packages/runtime/src/traits/document.ts b/packages/runtime/src/traits/document.ts
new file mode 100644
index 00000000..d0d0bc3d
--- /dev/null
+++ b/packages/runtime/src/traits/document.ts
@@ -0,0 +1,34 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { trait } from 'koota';
+
+import type { Entity } from 'koota';
+
+/** The real DOM node owned by a scene node. */
+export type DomNode = Element | Text;
+
+/**
+ * One element of the document, and its place in it. `parent` and `children`
+ * are the tree the reconciler reads back: text nodes and element nodes in the
+ * one order they were inserted in, which is what the document answers
+ * `getFirstChild` and `getNextSibling` from.
+ */
+export interface SceneNode {
+ readonly entity: Entity;
+ /** Native composition elements participate in the Koota scene graph. */
+ readonly native: boolean;
+ tag: string;
+ props: Record<string, unknown>;
+ parent: SceneNode | null;
+ children: SceneNode[];
+ readonly domNode: DomNode | null;
+}
+
+// The document node an entity was rendered from, on the entity itself: the
+// one object the document and the renderer both hold for it, so `===` between
+// two of them means what it says. AoS storage like the other handle traits —
+// entity.get(Host) returns the node — and never serialized: a node is this
+// render's business, rebuilt whenever a document renders the entity again.
+export const Host = trait(() => null as SceneNode | null);
diff --git a/packages/runtime/src/traits/index.ts b/packages/runtime/src/traits/index.ts
index 5553c0a2..e54f947b 100644
--- a/packages/runtime/src/traits/index.ts
+++ b/packages/runtime/src/traits/index.ts
@@ -1,5 +1,6 @@
export * from './relations';
export * from './node';
+export * from './document';
export * from './transform';
export * from './style';
export * from './text';
diff --git a/packages/runtime/src/traits/media.ts b/packages/runtime/src/traits/media.ts
index 80bd135d..4b9f85db 100644
--- a/packages/runtime/src/traits/media.ts
+++ b/packages/runtime/src/traits/media.ts
@@ -4,10 +4,6 @@
import { trait } from 'koota';
-// Type-only imports: erased at compile time, so the traits ↔ media module
-// cycle never materializes at runtime.
-import type { HtmlHost } from '../media/html';
-import type { SurfaceHost } from '../media/surface';
import type { ShaderHost } from '../media/shader';
import type { ImageDecoder } from '../media/image';
import type { VideoDecoderInstance } from '../media/video';
@@ -22,14 +18,9 @@ export type MountData = {
textBox?: DocumentFragment;
};
-// Handle traits store live runtime instances (hosts, decoders, buses) and are
+// Handle traits store live runtime instances (decoders, buses, etc.) and are
// never serialized. They use AoS storage: entity.get(X) returns the instance
-// directly. Suffixed Handle so the trait and the class it stores (HtmlHost,
-// AudioDecoder, ...) can share one export surface.
-
-export const HtmlHostHandle = trait(() => null as HtmlHost | null);
-
-export const SurfaceHostHandle = trait(() => null as SurfaceHost | null);
+// directly.
export const ShaderHostHandle = trait(() => null as ShaderHost | null);
diff --git a/packages/runtime/src/traits/world.ts b/packages/runtime/src/traits/world.ts
index edf7fac2..87d50b50 100644
--- a/packages/runtime/src/traits/world.ts
+++ b/packages/runtime/src/traits/world.ts
@@ -48,8 +48,8 @@ export const Time = trait({ now: 0, delta: 0 });
// Project frame rate (frames per second).
export const FrameRate = trait({ value: 30 });
-// Render target injected by the host: the editor canvas in the app, an
-// OffscreenCanvas during capture. resolution is the device pixel ratio.
+// Render target injected by the host. Capture uses an HTML canvas too so HTML
+// paint roots can live in its layout subtree. resolution is the pixel ratio.
export const RenderSurface = trait({
canvas: () => null as HTMLCanvasElement | OffscreenCanvas | null,
ctx: () => null as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null,
diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.ts
index e48ac428..524694fe 100644
--- a/packages/runtime/src/world/observers.ts
+++ b/packages/runtime/src/world/observers.ts
@@ -26,7 +26,7 @@ import {
ColorStop, StrokeStyle, Size, Computed, Active, Stage, IsMask,
ImageDecoderHandle, VideoDecoderHandle,
AudioDecoderHandle, CaptionDecoderHandle, WaveformHandle,
- HtmlHostHandle, SurfaceHostHandle, ShaderHostHandle, AudioBusHandle,
+ ShaderHostHandle, AudioBusHandle,
} from '../traits';
import { getParentEntity } from '../queries/hierarchy';
import { evictFromCaches, rebuildCaches, refileMask } from '../actions/cache';
@@ -83,8 +83,6 @@ export function observeWorld(world: World): () => void {
subs.push(world.onRemove(AudioDecoderHandle, (e) => e.get(AudioDecoderHandle)?.reset()));
subs.push(world.onRemove(CaptionDecoderHandle, (e) => e.get(CaptionDecoderHandle)?.dispose()));
subs.push(world.onRemove(WaveformHandle, (e) => e.get(WaveformHandle)?.dispose()));
- subs.push(world.onRemove(HtmlHostHandle, (e) => e.get(HtmlHostHandle)?.dispose()));
- subs.push(world.onRemove(SurfaceHostHandle, (e) => e.get(SurfaceHostHandle)?.dispose()));
subs.push(world.onRemove(ShaderHostHandle, (e) => e.get(ShaderHostHandle)?.dispose()));
subs.push(world.onRemove(AudioBusHandle, (e) => e.get(AudioBusHandle)?.disconnect()));