Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add image source handling and readiness mechanism to HtmlHost

konstantin-paulus1539ded67a35
diff --git a/apps/web/src/components/engine/decoders/html.ts b/apps/web/src/components/engine/decoders/html.ts
index ab28f249..ba69eac7 100644
--- a/apps/web/src/components/engine/decoders/html.ts
+++ b/apps/web/src/components/engine/decoders/html.ts
@@ -102,6 +102,26 @@ export class HtmlHost {
this.element = el;
}
+ /**
+ * Resolves once this host is safe to rasterize: every `<img>` in its
+ * subtree has decoded (or failed), and only then does the rendering update
+ * that refreshes the paint records run. Offline renders await this per
+ * frame, so an image still in flight can't paint as an empty box, and a
+ * source that lands late still makes it into the records that frame.
+ */
+ public whenReady(): Promise<void> {
+ const decoding: Promise<unknown>[] = [];
+ for (const image of this.element.querySelectorAll('img')) {
+ // `complete` covers loaded, failed, and srcless images alike; the
+ // rest are still in flight, and a failed decode just paints nothing.
+ if (image.complete) continue;
+ decoding.push(image.decode().catch(() => undefined));
+ }
+
+ if (decoding.length === 0) return nextRenderingUpdate();
+ return Promise.all(decoding).then(() => nextRenderingUpdate());
+ }
+
public setSize(width: number, height: number): void {
if (width === this.width && height === this.height) return;
this.width = width;
diff --git a/apps/web/src/components/engine/systems/playback.ts b/apps/web/src/components/engine/systems/playback.ts
index f0a0ee4b..24a8fcd9 100644
--- a/apps/web/src/components/engine/systems/playback.ts
+++ b/apps/web/src/components/engine/systems/playback.ts
@@ -5,7 +5,6 @@
import { hasComponent, Not, Or, query } from 'bitecs';
import { ChildOf, PaintType } from '../components';
import { resolveAudioDecoder, resolveCaptionDecoder, resolveImageDecoder, resolveSequenceDecoder, resolveShaderHost, resolveVideoDecoder } from '../decoders';
-import { nextRenderingUpdate } from '../decoders/html';
import { getParentEntity } from '../api';
import { getTransitionWindow } from '../utils/transition';
import { AudioBus } from '../services/audio-bus';
@@ -211,7 +210,7 @@ function forwardAudioDecoder(world: EngineWorld, sid: number, eid: number, aid?:
function forwardHtmlHost(world: EngineWorld, eid: number, fid: number): void {
const c = world.components;
if (world.mode === 'realtime' || c.Computed.visibility[eid] !== 1 || !c.HtmlHost[fid]) return;
- world.promises?.push(nextRenderingUpdate());
+ world.promises?.push(c.HtmlHost[fid].whenReady());
}
function forwardImageDecoder(world: EngineWorld, _sid: number, _eid: number, fid: number): void {
diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.ts
index 09715b6e..e08e434a 100644
--- a/apps/web/src/utils/jsx.ts
+++ b/apps/web/src/utils/jsx.ts
@@ -27,6 +27,7 @@ import {
deleteEntity,
findAssetDuration,
findGeometryAsset,
+ getAssetFile,
getEntityTree,
getNextName,
getParentEntity,
@@ -82,6 +83,10 @@ function timingProp(props: Record<string, unknown> | undefined, name: "start" |
const SVG_NS = "http://www.w3.org/2000/svg";
+// `src` values an html `<img>` carries straight to the browser; anything else
+// is a composition source the host has to resolve first.
+const DIRECT_SRC_RE = /^(?:data|blob):/i;
+
// Multiple new scenes are laid side by side in render order.
const PLACEMENT_GAP = 40;
@@ -391,6 +396,14 @@ export class WorldDocument implements ProjectDocument<HostNode> {
// wrapper for <Html>, the paint entity for <HtmlPaint>).
private htmlHosts = new Map<number, HtmlHost>();
+ // Object URLs minted for host-resolved <img> sources: a reactive `src` swap
+ // frees the one its element held, and teardown frees whatever is left.
+ private objectUrls = new Map<HTMLImageElement, string>();
+
+ // Per-element request counter, so a slow resolution that a newer `src` has
+ // already superseded doesn't land on top of it.
+ private imageEpochs = new WeakMap<HTMLImageElement, number>();
+
// One stable handle per entity: Solid's reconciliation compares host nodes
// by object identity, so traversal must return the same instance that
// `createElement` handed out.
@@ -431,6 +444,8 @@ export class WorldDocument implements ProjectDocument<HostNode> {
public disposeHosts(): void {
for (const host of this.createdHosts) host.dispose();
this.createdHosts.length = 0;
+ for (const url of this.objectUrls.values()) URL.revokeObjectURL(url);
+ this.objectUrls.clear();
}
// Set once `commit()` finishes. A mount keeps its reactive graph running
@@ -912,6 +927,10 @@ export class WorldDocument implements ProjectDocument<HostNode> {
if (name === "children" || name === "ref" || value === undefined || node instanceof Text) return;
if (node instanceof Element) {
+ if (name === "src" && node instanceof HTMLImageElement) {
+ this.setImageSource(node, value);
+ return;
+ }
setDomProperty(node, name, value);
return;
}
@@ -1831,6 +1850,80 @@ export class WorldDocument implements ProjectDocument<HostNode> {
});
}
+ /**
+ * `src` of an `<img>` inside html content. It takes the same inputs as a
+ * composition `src` — a path, an asset id, a URL, or a `generate.*` ref —
+ * which the module can't read itself, so they resolve through the host into
+ * an object URL. Sources the browser can already fetch (`data:`, `blob:`)
+ * are set as they are.
+ */
+ private setImageSource(el: HTMLImageElement, value: unknown): void {
+ const epoch = (this.imageEpochs.get(el) ?? 0) + 1;
+ this.imageEpochs.set(el, epoch);
+
+ if (typeof value === "string" && DIRECT_SRC_RE.test(value)) {
+ this.pointImageAt(el, value, false);
+ return;
+ }
+
+ // An empty `src` is the placeholder a signal reads before its source
+ // exists, so it clears the element rather than failing the render.
+ if (typeof value === "string" && value.trim().length === 0) {
+ this.releaseImageUrl(el);
+ el.removeAttribute("src");
+ return;
+ }
+
+ assert(
+ isAssetRef(value) || typeof value === "string",
+ "an <img> `src` must be a path, URL, asset id, or `generate.*` ref" + `, value: ${value}`,
+ );
+
+ this.trackImageLoad((async () => {
+ const input = value as AssetInput;
+ const asset = isAssetRef(input)
+ ? await resolveGeneratedAsset(this.world, input, new Map())
+ : await resolveAsset(this.world, input);
+ const file = await getAssetFile(asset);
+
+ // A newer `src` claimed this element while the old one resolved.
+ if (this.imageEpochs.get(el) !== epoch) return;
+
+ this.pointImageAt(el, URL.createObjectURL(file), true);
+ // Decoded here rather than at draw time so the first frame that samples
+ // this subtree already has pixels; a broken source just paints nothing.
+ await el.decode().catch(() => undefined);
+ })());
+ }
+
+ /** Points the element at `url`, freeing the object URL it held before. */
+ private pointImageAt(el: HTMLImageElement, url: string, owned: boolean): void {
+ this.releaseImageUrl(el);
+ if (owned) this.objectUrls.set(el, url);
+ el.src = url;
+ }
+
+ /** Frees the object URL this element holds, if it minted one. */
+ private releaseImageUrl(el: HTMLImageElement): void {
+ const url = this.objectUrls.get(el);
+ if (url === undefined) return;
+ URL.revokeObjectURL(url);
+ this.objectUrls.delete(el);
+ }
+
+ /**
+ * Makes an image source part of what a render waits for: `commit()` for the
+ * initial mount, so export and capture have it by frame 0, and the world's
+ * promise queue, which offline renders drain before every frame — that is
+ * what catches a live mount swapping `src` mid-timeline. Only the commit
+ * copy rejects; a source that breaks later must not take an export down.
+ */
+ private trackImageLoad(promise: Promise<void>): void {
+ this.world.promises?.push(promise.catch(() => undefined));
+ if (this.committed) promise.catch(console.error);
+ else this.promises.push(promise);
+ }
+
}
/**
diff --git a/packages/jsx/src/jsx-runtime.ts b/packages/jsx/src/jsx-runtime.ts
index 16f807f2..4681a7ed 100644
--- a/packages/jsx/src/jsx-runtime.ts
+++ b/packages/jsx/src/jsx-runtime.ts
@@ -19,6 +19,7 @@
*/
import type { JSX as SolidJSX } from "solid-js";
+import type { AssetInput } from "./generate";
import type {
AudioProps,
CaptionsProps,
@@ -43,7 +44,12 @@ import type {
// <surface> instead). "audio", "video", and "html" are dropped as DOM tags:
// media doesn't play under a paint host and a nested <html> is meaningless,
// so those names belong to the composition elements below.
-type HtmlElementTags = Omit<SolidJSX.HTMLElementTags, "canvas" | "audio" | "video" | "html">;
+type HtmlElementTags = Omit<SolidJSX.HTMLElementTags, "canvas" | "audio" | "video" | "html" | "img">;
+
+// `<img>` keeps every DOM attribute but takes the composition `src` inputs:
+// a path, an asset id, a URL, or a `generate.*` ref all resolve through the
+// host, and `data:`/`blob:` sources pass through to the browser untouched.
+type ImgTag = Omit<SolidJSX.HTMLElementTags["img"], "src"> & { src?: AssetInput };
// The shared names are re-declared below as unions with the composition props.
type SvgElementTags = Omit<SolidJSX.SVGElementTags, "rect" | "text" | "image">;
@@ -58,6 +64,7 @@ export declare namespace JSX {
}
export interface IntrinsicElements extends HtmlElementTags, SvgElementTags {
+ img: ImgTag;
group: GroupProps;
rect: RectProps | SolidJSX.SVGElementTags["rect"];
video: VideoProps;
diff --git a/reference/jsx/fonts.md b/reference/jsx/fonts.md
index 79a84227..503691c7 100644
--- a/reference/jsx/fonts.md
+++ b/reference/jsx/fonts.md
@@ -22,7 +22,7 @@ In [`<html>`](./html.md) you style text with ordinary CSS, so **use a locally in
</html>
```
-Remote web fonts do not work: a Google Fonts `<link>` or an `@font-face` with a `url()` source is a cross-origin subresource, and those are excluded from the painted read-back (the same rule that drops remote images, see [html.md](./html.md#requirements-and-limitations)). The glyphs would render on screen but come out blank in `dapi node capture` and exports. Stay on families from [`dapi fonts`](../fonts.md).
+Remote web fonts (a Google Fonts `<link>`, or an `@font-face` with a `url()` source) do render, but nothing waits for the fetch. Stay on local families from [`dapi fonts`](../fonts.md) and the output is deterministic.
If you want to pin an exact variant, declare an `@font-face` whose source is the CSS `local()` string that `dapi fonts` reports for that variant (never a `url()`):
diff --git a/reference/jsx/html.md b/reference/jsx/html.md
index 613368bc..61485632 100644
--- a/reference/jsx/html.md
+++ b/reference/jsx/html.md
@@ -51,6 +51,10 @@ setInterval(() => setCount((c) => c + 1), 1000);
</html>
```
+## Images
+
+`<img>` takes the same sources as a composition [`src`](./media.md): a path, an asset id, a URL, or a [`generate.*`](./generate.md) ref. The host resolves them exactly as it does for [`<image>`](./image.md)
+
## Props
`<html>` takes all [common props](./elements.md#common-props). `<htmlPaint>` takes:
@@ -68,5 +72,4 @@ The compiled module is persisted with the document: on reload, export, and `dapi
## Requirements and limitations
- `<audio>` and `<video>` tags are rejected: media doesn't play under a paint host. Use the [`<audio>`](./audio.md) and [`<video>`](./video.md) composition elements, which own playback and the timeline.
-- Cross-origin subresources (e.g. remote images, remote web fonts) are excluded from the painted output by the browser's read-back rules; use local assets and locally installed fonts (see [fonts.md](./fonts.md)).
- **`<html>` is sourceless, so with no `end` it defaults to a 16-second duration and disappears after 16 s** — a silent cutoff with no error.
diff --git a/reference/jsx/media.md b/reference/jsx/media.md
index 06a827c0..50db6107 100644
--- a/reference/jsx/media.md
+++ b/reference/jsx/media.md
@@ -1,6 +1,6 @@
# Media source resolution
-`src` (on [`<video>`](./video.md), [`<image>`](./image.md), [`<audio>`](./audio.md)) accepts:
+`src` (on [`<video>`](./video.md), [`<image>`](./image.md), [`<audio>`](./audio.md), and the DOM [`<img>`](./html.md#images) inside `<html>`) accepts:
- **Global path**: e.g. `"/Movies/video.mp4"`, resolved against the user's OS.
- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, registered as a remote asset.
@@ -9,4 +9,6 @@
In [`dapi node patch`](../node/patch.md), `src` takes a path, URL, or asset id; `generate.*` declarations are JSX-only.
+An `<img>` additionally takes a `data:` or `blob:` URL, which goes to the browser as it is.
+
To read a source's raw bytes inside an effect (rather than mount it as a node), pass the same input to [`useFile`](./lifecycle.md#usefile), which resolves it to a `File`.