The VS Code of video editing, built for agents.
TypeScript
Implement HTML host disposal and sharing functionality
konstantin-paulusb9dd3a71b5eb
diff --git a/apps/web/src/components/engine/api/observers.ts b/apps/web/src/components/engine/api/observers.tsindex 55a5112c..90a1353f 100644--- a/apps/web/src/components/engine/api/observers.ts+++ b/apps/web/src/components/engine/api/observers.ts@@ -481,6 +481,7 @@ export function observeWorld(world: EngineWorld) {utils.rebuildCaches(world, eid, getParentEntity(world, eid));utils.reactToChildDetached(world, eid);utils.disposeDecoders(world, eid);+ utils.disposeHtmlHosts(world, eid);utils.disconnectAudioBus(world, eid);utils.unpersistEntity(world, eid);world.rebuildTimelineIndex();diff --git a/apps/web/src/components/engine/api/utils.ts b/apps/web/src/components/engine/api/utils.tsindex 4a99a75d..6bd1a06b 100644--- a/apps/web/src/components/engine/api/utils.ts+++ b/apps/web/src/components/engine/api/utils.ts@@ -134,10 +134,7 @@ export function disposeDecoders(world: EngineWorld, eid: number): void {const VideoDecoder = world.components.VideoDecoder;const SequenceDecoder = world.components.SequenceDecoder;const AudioDecoder = world.components.AudioDecoder;- const HtmlHost = world.components.HtmlHost;- HtmlHost[eid]?.dispose();- HtmlHost[eid] = null;ImageDecoder[eid]?.dispose();ImageDecoder[eid] = null;VideoDecoder[eid]?.dispose();@@ -152,6 +149,17 @@ export function disposeDecoders(world: EngineWorld, eid: number): void {}}+export function disposeHtmlHosts(world: EngineWorld, eid: number): void {+ const HtmlHost = world.components.HtmlHost;++ HtmlHost[eid]?.dispose();+ HtmlHost[eid] = null;++ for (const child of query(world, [ChildOf(eid)])) {+ disposeHtmlHosts(world, child);+ }+}+export function disconnectAudioBus(world: EngineWorld, eid: number): void {const AudioBus = world.components.AudioBus;AudioBus[eid]?.disconnect();diff --git a/apps/web/src/components/engine/decoders/html.ts b/apps/web/src/components/engine/decoders/html.tsindex 87d6f3d6..cc32a38e 100644--- a/apps/web/src/components/engine/decoders/html.ts+++ b/apps/web/src/components/engine/decoders/html.ts@@ -2,6 +2,8 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */+import { addComponent } from '../api/events';+import type { EngineWorld } from '../api/world';type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;@@ -13,56 +15,46 @@ type DrawElementContext = Ctx2D & {drawElementImage(source: Element | unknown, dx: number, dy: number, dw: number, dh: number): DOMMatrix;};-type CaptureCanvas = HTMLCanvasElement & {- captureElementImage?(element: Element): unknown;-};-export function isHtmlInCanvasSupported(): boolean {return typeof CanvasRenderingContext2D !== 'undefined'&& 'drawElementImage' in CanvasRenderingContext2D.prototype;}-// drawElementImage only accepts direct children of a canvas carrying the-// `layoutsubtree` attribute. Hosts normally live under the visible stage-// canvas and are drawn directly; this hidden singleton is the fallback-// parent when the world has no DOM canvas, drawable elsewhere only as an-// ElementImage snapshot via captureElementImage.-let layoutCanvas: HTMLCanvasElement | null = null;--function getLayoutCanvas(): HTMLCanvasElement {- if (!layoutCanvas) {- layoutCanvas = document.createElement('canvas');- layoutCanvas.width = 1;- layoutCanvas.height = 1;- layoutCanvas.style.cssText = 'position:fixed;left:0;top:0;visibility:hidden;pointer-events:none;';- document.body.appendChild(layoutCanvas);- }- return layoutCanvas;-}-/**- * DOM host for one HTML paint: a div child of a layoutsubtree canvas. The- * mount document inserts the paint's JSX children into `element` as real,- * reactive DOM; the browser lays them out at the parent geometry's box size- * and `draw` paints the result each frame.+ * DOM host for one HTML paint: a div inside the host's own layoutsubtree+ * canvas. The mount document inserts the paint's JSX children into `element`+ * as real, reactive DOM; the browser lays them out at the parent geometry's+ * box size and `draw` rasterizes the result onto the host canvas and blits+ * it into the rendering context (stage or offscreen alike).+ *+ * The host canvas must stay inside the viewport: the browser only caches its+ * children's paint records (which drawElementImage reads) while the canvas+ * itself paints, so hiding it or parking it offscreen breaks every draw. The+ * 1px CSS size keeps it invisible in practice while its bitmap is resized+ * freely for rasterization.*/export class HtmlHost {public readonly element: HTMLDivElement;- private readonly canvas: HTMLCanvasElement;+ private readonly canvas = document.createElement('canvas');+ private readonly ctx = this.canvas.getContext('2d')!;private disposed = false;private warned = false;private width = -1;private height = -1;- public constructor(world: EngineWorld) {- this.canvas = world.canvas instanceof HTMLCanvasElement ? world.canvas : getLayoutCanvas();+ public constructor() {+ this.canvas.width = 1;+ this.canvas.height = 1;this.canvas.toggleAttribute('layoutsubtree', true);+ this.canvas.style.cssText = 'position:fixed;left:0;top:0;width:1px;height:1px;pointer-events:none;';const el = document.createElement('div');// pointer-events: none — layoutsubtree children participate in hit- // testing and would otherwise swallow stage interactions.+ // testing and would otherwise swallow page interactions.el.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';this.canvas.appendChild(el);+ document.body.appendChild(this.canvas);+this.element = el;}@@ -80,19 +72,24 @@ export class HtmlHost {this.setSize(width, height);try {- if (ctx.canvas === this.canvas) {- (ctx as DrawElementContext).drawElementImage(this.element, 0, 0, width, height);- } else {- const capture = (this.canvas as CaptureCanvas).captureElementImage;- if (typeof capture !== 'function') {- throw new Error('captureElementImage is not available; cannot draw <htmlPaint> content offscreen');- }- const image = capture.call(this.canvas, this.element);- (ctx as DrawElementContext).drawElementImage(image, 0, 0, width, height);+ const m = ctx.getTransform();+ const sx = Math.hypot(m.a, m.b) || 1;+ const sy = Math.hypot(m.c, m.d) || 1;+ const pw = Math.max(1, Math.ceil(width * sx));+ const ph = Math.max(1, Math.ceil(height * sy));+++ if (this.canvas.width !== pw || this.canvas.height !== ph) {+ this.canvas.width = pw;+ this.canvas.height = ph;}+ this.ctx.setTransform(sx, 0, 0, sy, 0, 0);+ this.ctx.clearRect(0, 0, width, height);+ (this.ctx as DrawElementContext).drawElementImage(this.element, 0, 0, width, height);+ ctx.drawImage(this.canvas, 0, 0, width, height);} catch (e) {- // A transient failure (subtree not yet laid out) just skips the- // frame; only log once so a permanent one doesn't spam per frame.+ // A transient failure (subtree not yet laid out or painted) just+ // skips the frame; log once so a permanent one doesn't spam.if (!this.warned) {this.warned = true;console.error('Error drawing <htmlPaint> content', e);@@ -103,6 +100,26 @@ export class HtmlHost {public dispose(): void {if (this.disposed) return;this.disposed = true;- this.element.remove();+ this.canvas.remove();+ }+}++/**+ * Shares <htmlPaint> hosts with a cloned (offline) world. Host DOM is+ * runtime state serializeEntity cannot carry, so export/capture worlds+ * reference the live hosts directly and sample whatever the mount's reactive+ * graph shows while encoding. The source world keeps ownership; offline+ * worlds must not dispose them.+ */+export function shareHtmlHosts(+ sourceWorld: EngineWorld,+ world: EngineWorld,+ eidMap: Map<number, number>,+) {+ for (const [sourceEid, targetEid] of eidMap) {+ const host = sourceWorld.components.HtmlHost[sourceEid];+ if (!host) continue;+ addComponent(world, targetEid, world.components.HtmlHost, false);+ world.components.HtmlHost[targetEid] = host;}}diff --git a/apps/web/src/components/engine/encode/encoder.ts b/apps/web/src/components/engine/encode/encoder.tsindex 56dcc5ca..d803faec 100644--- a/apps/web/src/components/engine/encode/encoder.ts+++ b/apps/web/src/components/engine/encode/encoder.ts@@ -24,6 +24,7 @@ import { playbackSystem } from '../systems/playback';import { motionSystem } from '../systems/motion';import { transformSystem } from '../systems/transform';import { renderSystem } from '../systems/render';+import { shareHtmlHosts } from '../decoders/html';import { cloneFromRecords, serializeEntity } from '../api/serialize';import { getEntityTree } from '../api/query';import { AudioBus } from '../services/audio-bus';@@ -131,6 +132,10 @@ export async function createEncoder(sourceWorld: EngineWorld, config: EncoderConc.AudioPlayback.contextOffsetInSeconds[sceneEid] = 0;c.AudioPlayback.timelineOffsetInSeconds[sceneEid] = playheadStartSeconds;+ if (videoEnabled) {+ shareHtmlHosts(sourceWorld, world, eidMap);+ }+// Set up mediabunny outputconst buffer = await TargetBuffer.create(config.target);const format = await createOutputFormat(buffer, config.format);diff --git a/apps/web/src/components/engine/encode/image-encoder.ts b/apps/web/src/components/engine/encode/image-encoder.tsindex 53808377..54ce2bd6 100644--- a/apps/web/src/components/engine/encode/image-encoder.ts+++ b/apps/web/src/components/engine/encode/image-encoder.ts@@ -15,6 +15,7 @@ import { motionSystem } from '../systems/motion';import { transformSystem } from '../systems/transform';import { renderSystem } from '../systems/render';import { cloneFromRecords, serializeEntity } from '../api/serialize';+import { shareHtmlHosts } from '../decoders/html';import { getEntityTree } from '../api/query';import { framesToSeconds, formatTimestamp, stampTimestampLabel } from '../utils';import { assert } from '@/utils';@@ -82,6 +83,8 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: Imageconst rootEid = eidMap.get(config.eid);assert(rootEid !== undefined, 'Failed to clone entity subtree');+ shareHtmlHosts(sourceWorld, world, eidMap);+const clonedEids = [...eidMap.values()];const c = world.components;diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.tsindex e4a4937a..bb3a5038 100644--- a/apps/web/src/utils/jsx.ts+++ b/apps/web/src/utils/jsx.ts@@ -561,7 +561,7 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {"<html> requires the html-in-canvas API; enable chrome://flags/#canvas-draw-element",);setComponent(world, eid, c.Paint, PaintType.HTML);- const host = new HtmlHost(world);+ const host = new HtmlHost();addComponent(world, eid, c.HtmlHost, false);c.HtmlHost[eid] = host;node.host = host;