Skip to content

The VS Code of video editing, built for agents.

TypeScript

Implement HTML host disposal and sharing functionality

konstantin-paulusb9dd3a71b5eb
diff --git a/apps/web/src/components/engine/api/observers.ts b/apps/web/src/components/engine/api/observers.ts
index 55a5112c..90a1353f 100644
--- a/apps/web/src/components/engine/api/observers.ts
+++ b/apps/web/src/components/engine/api/observers.ts
@@ -481,6 +481,7 @@ export function observeWorld(world: EngineWorld) {
utils.rebuildCaches(world, eid, getParentEntity(world, eid));
utils.reactToChildDetached(world, eid);
utils.disposeDecoders(world, eid);
+ utils.disposeHtmlHosts(world, eid);
utils.disconnectAudioBus(world, eid);
utils.unpersistEntity(world, eid);
world.rebuildTimelineIndex();
diff --git a/apps/web/src/components/engine/api/utils.ts b/apps/web/src/components/engine/api/utils.ts
index 4a99a75d..6bd1a06b 100644
--- a/apps/web/src/components/engine/api/utils.ts
+++ b/apps/web/src/components/engine/api/utils.ts
@@ -134,10 +134,7 @@ export function disposeDecoders(world: EngineWorld, eid: number): void {
const VideoDecoder = world.components.VideoDecoder;
const SequenceDecoder = world.components.SequenceDecoder;
const AudioDecoder = world.components.AudioDecoder;
- const HtmlHost = world.components.HtmlHost;
- HtmlHost[eid]?.dispose();
- HtmlHost[eid] = null;
ImageDecoder[eid]?.dispose();
ImageDecoder[eid] = null;
VideoDecoder[eid]?.dispose();
@@ -152,6 +149,17 @@ export function disposeDecoders(world: EngineWorld, eid: number): void {
}
}
+export function disposeHtmlHosts(world: EngineWorld, eid: number): void {
+ const HtmlHost = world.components.HtmlHost;
+
+ HtmlHost[eid]?.dispose();
+ HtmlHost[eid] = null;
+
+ for (const child of query(world, [ChildOf(eid)])) {
+ disposeHtmlHosts(world, child);
+ }
+}
+
export function disconnectAudioBus(world: EngineWorld, eid: number): void {
const AudioBus = world.components.AudioBus;
AudioBus[eid]?.disconnect();
diff --git a/apps/web/src/components/engine/decoders/html.ts b/apps/web/src/components/engine/decoders/html.ts
index 87d6f3d6..cc32a38e 100644
--- a/apps/web/src/components/engine/decoders/html.ts
+++ b/apps/web/src/components/engine/decoders/html.ts
@@ -2,6 +2,8 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+import { addComponent } from '../api/events';
+
import type { EngineWorld } from '../api/world';
type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
@@ -13,56 +15,46 @@ type DrawElementContext = Ctx2D & {
drawElementImage(source: Element | unknown, dx: number, dy: number, dw: number, dh: number): DOMMatrix;
};
-type CaptureCanvas = HTMLCanvasElement & {
- captureElementImage?(element: Element): unknown;
-};
-
export function isHtmlInCanvasSupported(): boolean {
return typeof CanvasRenderingContext2D !== 'undefined'
&& 'drawElementImage' in CanvasRenderingContext2D.prototype;
}
-// drawElementImage only accepts direct children of a canvas carrying the
-// `layoutsubtree` attribute. Hosts normally live under the visible stage
-// canvas and are drawn directly; this hidden singleton is the fallback
-// parent when the world has no DOM canvas, drawable elsewhere only as an
-// ElementImage snapshot via captureElementImage.
-let layoutCanvas: HTMLCanvasElement | null = null;
-
-function getLayoutCanvas(): HTMLCanvasElement {
- if (!layoutCanvas) {
- layoutCanvas = document.createElement('canvas');
- layoutCanvas.width = 1;
- layoutCanvas.height = 1;
- layoutCanvas.style.cssText = 'position:fixed;left:0;top:0;visibility:hidden;pointer-events:none;';
- document.body.appendChild(layoutCanvas);
- }
- return layoutCanvas;
-}
-
/**
- * DOM host for one HTML paint: a div child of a layoutsubtree canvas. The
- * mount document inserts the paint's JSX children into `element` as real,
- * reactive DOM; the browser lays them out at the parent geometry's box size
- * and `draw` paints the result each frame.
+ * DOM host for one HTML paint: a div inside the host's own layoutsubtree
+ * canvas. The mount document inserts the paint's JSX children into `element`
+ * as real, reactive DOM; the browser lays them out at the parent geometry's
+ * box size and `draw` rasterizes the result onto the host canvas and blits
+ * it into the rendering context (stage or offscreen alike).
+ *
+ * The host canvas must stay inside the viewport: the browser only caches its
+ * children's paint records (which drawElementImage reads) while the canvas
+ * itself paints, so hiding it or parking it offscreen breaks every draw. The
+ * 1px CSS size keeps it invisible in practice while its bitmap is resized
+ * freely for rasterization.
*/
export class HtmlHost {
public readonly element: HTMLDivElement;
- private readonly canvas: HTMLCanvasElement;
+ private readonly canvas = document.createElement('canvas');
+ private readonly ctx = this.canvas.getContext('2d')!;
private disposed = false;
private warned = false;
private width = -1;
private height = -1;
- public constructor(world: EngineWorld) {
- this.canvas = world.canvas instanceof HTMLCanvasElement ? world.canvas : getLayoutCanvas();
+ public constructor() {
+ this.canvas.width = 1;
+ this.canvas.height = 1;
this.canvas.toggleAttribute('layoutsubtree', true);
+ this.canvas.style.cssText = 'position:fixed;left:0;top:0;width:1px;height:1px;pointer-events:none;';
const el = document.createElement('div');
// pointer-events: none — layoutsubtree children participate in hit
- // testing and would otherwise swallow stage interactions.
+ // testing and would otherwise swallow page interactions.
el.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';
this.canvas.appendChild(el);
+ document.body.appendChild(this.canvas);
+
this.element = el;
}
@@ -80,19 +72,24 @@ export class HtmlHost {
this.setSize(width, height);
try {
- if (ctx.canvas === this.canvas) {
- (ctx as DrawElementContext).drawElementImage(this.element, 0, 0, width, height);
- } else {
- const capture = (this.canvas as CaptureCanvas).captureElementImage;
- if (typeof capture !== 'function') {
- throw new Error('captureElementImage is not available; cannot draw <htmlPaint> content offscreen');
- }
- const image = capture.call(this.canvas, this.element);
- (ctx as DrawElementContext).drawElementImage(image, 0, 0, width, height);
+ const m = ctx.getTransform();
+ const sx = Math.hypot(m.a, m.b) || 1;
+ const sy = Math.hypot(m.c, m.d) || 1;
+ const pw = Math.max(1, Math.ceil(width * sx));
+ const ph = Math.max(1, Math.ceil(height * sy));
+
+
+ if (this.canvas.width !== pw || this.canvas.height !== ph) {
+ this.canvas.width = pw;
+ this.canvas.height = ph;
}
+ this.ctx.setTransform(sx, 0, 0, sy, 0, 0);
+ this.ctx.clearRect(0, 0, width, height);
+ (this.ctx as DrawElementContext).drawElementImage(this.element, 0, 0, width, height);
+ ctx.drawImage(this.canvas, 0, 0, width, height);
} catch (e) {
- // A transient failure (subtree not yet laid out) just skips the
- // frame; only log once so a permanent one doesn't spam per frame.
+ // A transient failure (subtree not yet laid out or painted) just
+ // skips the frame; log once so a permanent one doesn't spam.
if (!this.warned) {
this.warned = true;
console.error('Error drawing <htmlPaint> content', e);
@@ -103,6 +100,26 @@ export class HtmlHost {
public dispose(): void {
if (this.disposed) return;
this.disposed = true;
- this.element.remove();
+ this.canvas.remove();
+ }
+}
+
+/**
+ * Shares <htmlPaint> hosts with a cloned (offline) world. Host DOM is
+ * runtime state serializeEntity cannot carry, so export/capture worlds
+ * reference the live hosts directly and sample whatever the mount's reactive
+ * graph shows while encoding. The source world keeps ownership; offline
+ * worlds must not dispose them.
+ */
+export function shareHtmlHosts(
+ sourceWorld: EngineWorld,
+ world: EngineWorld,
+ eidMap: Map<number, number>,
+) {
+ for (const [sourceEid, targetEid] of eidMap) {
+ const host = sourceWorld.components.HtmlHost[sourceEid];
+ if (!host) continue;
+ addComponent(world, targetEid, world.components.HtmlHost, false);
+ world.components.HtmlHost[targetEid] = host;
}
}
diff --git a/apps/web/src/components/engine/encode/encoder.ts b/apps/web/src/components/engine/encode/encoder.ts
index 56dcc5ca..d803faec 100644
--- a/apps/web/src/components/engine/encode/encoder.ts
+++ b/apps/web/src/components/engine/encode/encoder.ts
@@ -24,6 +24,7 @@ import { playbackSystem } from '../systems/playback';
import { motionSystem } from '../systems/motion';
import { transformSystem } from '../systems/transform';
import { renderSystem } from '../systems/render';
+import { shareHtmlHosts } from '../decoders/html';
import { cloneFromRecords, serializeEntity } from '../api/serialize';
import { getEntityTree } from '../api/query';
import { AudioBus } from '../services/audio-bus';
@@ -131,6 +132,10 @@ export async function createEncoder(sourceWorld: EngineWorld, config: EncoderCon
c.AudioPlayback.contextOffsetInSeconds[sceneEid] = 0;
c.AudioPlayback.timelineOffsetInSeconds[sceneEid] = playheadStartSeconds;
+ if (videoEnabled) {
+ shareHtmlHosts(sourceWorld, world, eidMap);
+ }
+
// Set up mediabunny output
const buffer = await TargetBuffer.create(config.target);
const format = await createOutputFormat(buffer, config.format);
diff --git a/apps/web/src/components/engine/encode/image-encoder.ts b/apps/web/src/components/engine/encode/image-encoder.ts
index 53808377..54ce2bd6 100644
--- a/apps/web/src/components/engine/encode/image-encoder.ts
+++ b/apps/web/src/components/engine/encode/image-encoder.ts
@@ -15,6 +15,7 @@ import { motionSystem } from '../systems/motion';
import { transformSystem } from '../systems/transform';
import { renderSystem } from '../systems/render';
import { cloneFromRecords, serializeEntity } from '../api/serialize';
+import { shareHtmlHosts } from '../decoders/html';
import { getEntityTree } from '../api/query';
import { framesToSeconds, formatTimestamp, stampTimestampLabel } from '../utils';
import { assert } from '@/utils';
@@ -82,6 +83,8 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: Image
const rootEid = eidMap.get(config.eid);
assert(rootEid !== undefined, 'Failed to clone entity subtree');
+ shareHtmlHosts(sourceWorld, world, eidMap);
+
const clonedEids = [...eidMap.values()];
const c = world.components;
diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.ts
index e4a4937a..bb3a5038 100644
--- a/apps/web/src/utils/jsx.ts
+++ b/apps/web/src/utils/jsx.ts
@@ -561,7 +561,7 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
"<html> requires the html-in-canvas API; enable chrome://flags/#canvas-draw-element",
);
setComponent(world, eid, c.Paint, PaintType.HTML);
- const host = new HtmlHost(world);
+ const host = new HtmlHost();
addComponent(world, eid, c.HtmlHost, false);
c.HtmlHost[eid] = host;
node.host = host;