The VS Code of video editing, built for agents.
TypeScript
Add image source handling and readiness mechanism to HtmlHost
konstantin-paulus1539ded67a35
diff --git a/apps/web/src/components/engine/decoders/html.ts b/apps/web/src/components/engine/decoders/html.tsindex ab28f249..ba69eac7 100644--- a/apps/web/src/components/engine/decoders/html.ts+++ b/apps/web/src/components/engine/decoders/html.ts@@ -102,6 +102,26 @@ export class HtmlHost {this.element = el;}+ /**+ * Resolves once this host is safe to rasterize: every `<img>` in its+ * subtree has decoded (or failed), and only then does the rendering update+ * that refreshes the paint records run. Offline renders await this per+ * frame, so an image still in flight can't paint as an empty box, and a+ * source that lands late still makes it into the records that frame.+ */+ public whenReady(): Promise<void> {+ const decoding: Promise<unknown>[] = [];+ for (const image of this.element.querySelectorAll('img')) {+ // `complete` covers loaded, failed, and srcless images alike; the+ // rest are still in flight, and a failed decode just paints nothing.+ if (image.complete) continue;+ decoding.push(image.decode().catch(() => undefined));+ }++ if (decoding.length === 0) return nextRenderingUpdate();+ return Promise.all(decoding).then(() => nextRenderingUpdate());+ }+public setSize(width: number, height: number): void {if (width === this.width && height === this.height) return;this.width = width;diff --git a/apps/web/src/components/engine/systems/playback.ts b/apps/web/src/components/engine/systems/playback.tsindex f0a0ee4b..24a8fcd9 100644--- a/apps/web/src/components/engine/systems/playback.ts+++ b/apps/web/src/components/engine/systems/playback.ts@@ -5,7 +5,6 @@import { hasComponent, Not, Or, query } from 'bitecs';import { ChildOf, PaintType } from '../components';import { resolveAudioDecoder, resolveCaptionDecoder, resolveImageDecoder, resolveSequenceDecoder, resolveShaderHost, resolveVideoDecoder } from '../decoders';-import { nextRenderingUpdate } from '../decoders/html';import { getParentEntity } from '../api';import { getTransitionWindow } from '../utils/transition';import { AudioBus } from '../services/audio-bus';@@ -211,7 +210,7 @@ function forwardAudioDecoder(world: EngineWorld, sid: number, eid: number, aid?:function forwardHtmlHost(world: EngineWorld, eid: number, fid: number): void {const c = world.components;if (world.mode === 'realtime' || c.Computed.visibility[eid] !== 1 || !c.HtmlHost[fid]) return;- world.promises?.push(nextRenderingUpdate());+ world.promises?.push(c.HtmlHost[fid].whenReady());}function forwardImageDecoder(world: EngineWorld, _sid: number, _eid: number, fid: number): void {diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.tsindex 09715b6e..e08e434a 100644--- a/apps/web/src/utils/jsx.ts+++ b/apps/web/src/utils/jsx.ts@@ -27,6 +27,7 @@ import {deleteEntity,findAssetDuration,findGeometryAsset,+ getAssetFile,getEntityTree,getNextName,getParentEntity,@@ -82,6 +83,10 @@ function timingProp(props: Record<string, unknown> | undefined, name: "start" |const SVG_NS = "http://www.w3.org/2000/svg";+// `src` values an html `<img>` carries straight to the browser; anything else+// is a composition source the host has to resolve first.+const DIRECT_SRC_RE = /^(?:data|blob):/i;+// Multiple new scenes are laid side by side in render order.const PLACEMENT_GAP = 40;@@ -391,6 +396,14 @@ export class WorldDocument implements ProjectDocument<HostNode> {// wrapper for <Html>, the paint entity for <HtmlPaint>).private htmlHosts = new Map<number, HtmlHost>();+ // Object URLs minted for host-resolved <img> sources: a reactive `src` swap+ // frees the one its element held, and teardown frees whatever is left.+ private objectUrls = new Map<HTMLImageElement, string>();++ // Per-element request counter, so a slow resolution that a newer `src` has+ // already superseded doesn't land on top of it.+ private imageEpochs = new WeakMap<HTMLImageElement, number>();+// One stable handle per entity: Solid's reconciliation compares host nodes// by object identity, so traversal must return the same instance that// `createElement` handed out.@@ -431,6 +444,8 @@ export class WorldDocument implements ProjectDocument<HostNode> {public disposeHosts(): void {for (const host of this.createdHosts) host.dispose();this.createdHosts.length = 0;+ for (const url of this.objectUrls.values()) URL.revokeObjectURL(url);+ this.objectUrls.clear();}// Set once `commit()` finishes. A mount keeps its reactive graph running@@ -912,6 +927,10 @@ export class WorldDocument implements ProjectDocument<HostNode> {if (name === "children" || name === "ref" || value === undefined || node instanceof Text) return;if (node instanceof Element) {+ if (name === "src" && node instanceof HTMLImageElement) {+ this.setImageSource(node, value);+ return;+ }setDomProperty(node, name, value);return;}@@ -1831,6 +1850,80 @@ export class WorldDocument implements ProjectDocument<HostNode> {});}+ /**+ * `src` of an `<img>` inside html content. It takes the same inputs as a+ * composition `src` — a path, an asset id, a URL, or a `generate.*` ref —+ * which the module can't read itself, so they resolve through the host into+ * an object URL. Sources the browser can already fetch (`data:`, `blob:`)+ * are set as they are.+ */+ private setImageSource(el: HTMLImageElement, value: unknown): void {+ const epoch = (this.imageEpochs.get(el) ?? 0) + 1;+ this.imageEpochs.set(el, epoch);++ if (typeof value === "string" && DIRECT_SRC_RE.test(value)) {+ this.pointImageAt(el, value, false);+ return;+ }++ // An empty `src` is the placeholder a signal reads before its source+ // exists, so it clears the element rather than failing the render.+ if (typeof value === "string" && value.trim().length === 0) {+ this.releaseImageUrl(el);+ el.removeAttribute("src");+ return;+ }++ assert(+ isAssetRef(value) || typeof value === "string",+ "an <img> `src` must be a path, URL, asset id, or `generate.*` ref" + `, value: ${value}`,+ );++ this.trackImageLoad((async () => {+ const input = value as AssetInput;+ const asset = isAssetRef(input)+ ? await resolveGeneratedAsset(this.world, input, new Map())+ : await resolveAsset(this.world, input);+ const file = await getAssetFile(asset);++ // A newer `src` claimed this element while the old one resolved.+ if (this.imageEpochs.get(el) !== epoch) return;++ this.pointImageAt(el, URL.createObjectURL(file), true);+ // Decoded here rather than at draw time so the first frame that samples+ // this subtree already has pixels; a broken source just paints nothing.+ await el.decode().catch(() => undefined);+ })());+ }++ /** Points the element at `url`, freeing the object URL it held before. */+ private pointImageAt(el: HTMLImageElement, url: string, owned: boolean): void {+ this.releaseImageUrl(el);+ if (owned) this.objectUrls.set(el, url);+ el.src = url;+ }++ /** Frees the object URL this element holds, if it minted one. */+ private releaseImageUrl(el: HTMLImageElement): void {+ const url = this.objectUrls.get(el);+ if (url === undefined) return;+ URL.revokeObjectURL(url);+ this.objectUrls.delete(el);+ }++ /**+ * Makes an image source part of what a render waits for: `commit()` for the+ * initial mount, so export and capture have it by frame 0, and the world's+ * promise queue, which offline renders drain before every frame — that is+ * what catches a live mount swapping `src` mid-timeline. Only the commit+ * copy rejects; a source that breaks later must not take an export down.+ */+ private trackImageLoad(promise: Promise<void>): void {+ this.world.promises?.push(promise.catch(() => undefined));+ if (this.committed) promise.catch(console.error);+ else this.promises.push(promise);+ }+}/**diff --git a/packages/jsx/src/jsx-runtime.ts b/packages/jsx/src/jsx-runtime.tsindex 16f807f2..4681a7ed 100644--- a/packages/jsx/src/jsx-runtime.ts+++ b/packages/jsx/src/jsx-runtime.ts@@ -19,6 +19,7 @@*/import type { JSX as SolidJSX } from "solid-js";+import type { AssetInput } from "./generate";import type {AudioProps,CaptionsProps,@@ -43,7 +44,12 @@ import type {// <surface> instead). "audio", "video", and "html" are dropped as DOM tags:// media doesn't play under a paint host and a nested <html> is meaningless,// so those names belong to the composition elements below.-type HtmlElementTags = Omit<SolidJSX.HTMLElementTags, "canvas" | "audio" | "video" | "html">;+type HtmlElementTags = Omit<SolidJSX.HTMLElementTags, "canvas" | "audio" | "video" | "html" | "img">;++// `<img>` keeps every DOM attribute but takes the composition `src` inputs:+// a path, an asset id, a URL, or a `generate.*` ref all resolve through the+// host, and `data:`/`blob:` sources pass through to the browser untouched.+type ImgTag = Omit<SolidJSX.HTMLElementTags["img"], "src"> & { src?: AssetInput };// The shared names are re-declared below as unions with the composition props.type SvgElementTags = Omit<SolidJSX.SVGElementTags, "rect" | "text" | "image">;@@ -58,6 +64,7 @@ export declare namespace JSX {}export interface IntrinsicElements extends HtmlElementTags, SvgElementTags {+ img: ImgTag;group: GroupProps;rect: RectProps | SolidJSX.SVGElementTags["rect"];video: VideoProps;diff --git a/reference/jsx/fonts.md b/reference/jsx/fonts.mdindex 79a84227..503691c7 100644--- a/reference/jsx/fonts.md+++ b/reference/jsx/fonts.md@@ -22,7 +22,7 @@ In [`<html>`](./html.md) you style text with ordinary CSS, so **use a locally in</html>```-Remote web fonts do not work: a Google Fonts `<link>` or an `@font-face` with a `url()` source is a cross-origin subresource, and those are excluded from the painted read-back (the same rule that drops remote images, see [html.md](./html.md#requirements-and-limitations)). The glyphs would render on screen but come out blank in `dapi node capture` and exports. Stay on families from [`dapi fonts`](../fonts.md).+Remote web fonts (a Google Fonts `<link>`, or an `@font-face` with a `url()` source) do render, but nothing waits for the fetch. Stay on local families from [`dapi fonts`](../fonts.md) and the output is deterministic.If you want to pin an exact variant, declare an `@font-face` whose source is the CSS `local()` string that `dapi fonts` reports for that variant (never a `url()`):diff --git a/reference/jsx/html.md b/reference/jsx/html.mdindex 613368bc..61485632 100644--- a/reference/jsx/html.md+++ b/reference/jsx/html.md@@ -51,6 +51,10 @@ setInterval(() => setCount((c) => c + 1), 1000);</html>```+## Images++`<img>` takes the same sources as a composition [`src`](./media.md): a path, an asset id, a URL, or a [`generate.*`](./generate.md) ref. The host resolves them exactly as it does for [`<image>`](./image.md)+## Props`<html>` takes all [common props](./elements.md#common-props). `<htmlPaint>` takes:@@ -68,5 +72,4 @@ The compiled module is persisted with the document: on reload, export, and `dapi## Requirements and limitations- `<audio>` and `<video>` tags are rejected: media doesn't play under a paint host. Use the [`<audio>`](./audio.md) and [`<video>`](./video.md) composition elements, which own playback and the timeline.-- Cross-origin subresources (e.g. remote images, remote web fonts) are excluded from the painted output by the browser's read-back rules; use local assets and locally installed fonts (see [fonts.md](./fonts.md)).- **`<html>` is sourceless, so with no `end` it defaults to a 16-second duration and disappears after 16 s** — a silent cutoff with no error.diff --git a/reference/jsx/media.md b/reference/jsx/media.mdindex 06a827c0..50db6107 100644--- a/reference/jsx/media.md+++ b/reference/jsx/media.md@@ -1,6 +1,6 @@# Media source resolution-`src` (on [`<video>`](./video.md), [`<image>`](./image.md), [`<audio>`](./audio.md)) accepts:+`src` (on [`<video>`](./video.md), [`<image>`](./image.md), [`<audio>`](./audio.md), and the DOM [`<img>`](./html.md#images) inside `<html>`) accepts:- **Global path**: e.g. `"/Movies/video.mp4"`, resolved against the user's OS.- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, registered as a remote asset.@@ -9,4 +9,6 @@In [`dapi node patch`](../node/patch.md), `src` takes a path, URL, or asset id; `generate.*` declarations are JSX-only.+An `<img>` additionally takes a `data:` or `blob:` URL, which goes to the browser as it is.+To read a source's raw bytes inside an effect (rather than mount it as a node), pass the same input to [`useFile`](./lifecycle.md#usefile), which resolves it to a `File`.