Skip to content

The VS Code of video editing, built for agents.

TypeScript

Update runtime package with new media handling features and dependencies

konstantin-paulus2774a07280f5
diff --git a/package-lock.json b/package-lock.json
index dfa6a387..90d8070c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -12331,11 +12331,13 @@
"license": "MPL-2.0",
"dependencies": {
"animejs": "^4.4.1",
+ "colord": "^2.9.3",
"koota": "^0.6.6",
"mediabunny": "^1.32.2",
"sqids": "^0.3.0"
},
"devDependencies": {
+ "@webgpu/types": "^0.1.71",
"typescript": "~5.9.3"
}
}
diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.md
index 43ea1351..2e1833a7 100644
--- a/packages/runtime/MIGRATION.md
+++ b/packages/runtime/MIGRATION.md
@@ -32,6 +32,11 @@ Working doc for migrating the editor engine from bitecs (in `apps/web/src/compon
| Actions: group, frame, clipboard | `src/actions/{group,frame,clipboard}.ts` | frame.ts also hosts `switchActiveScene` (was api/base.ts): sets the ActiveScene world trait, nothing else |
| Actions: assets, folders (model) | `src/actions/{assets,folders}.ts` | allocateId/raiseIdCounter, appendAssets/upsertAsset/removeAssets, getAssetFile/Blob, insertAssetInTimeline; folder tree ops return changed records for app persistence |
| Observers (runtime half) | `src/world/observers.ts` | koota subscriptions wiring cache upkeep, Computed mirrors, and time-range reactions; auto-registered by `createRuntimeWorld` (returns a disposer). See semantic notes |
+| Media: decoders + audio bus | `src/media/*` | frame-cache, keyframe-index, time-stretcher, surface, html, audio-peaks moved verbatim; image, video, sequence, audio, shader with entity-based resolvers; audio-bus koota-ported; dispose walkers in `src/media/dispose.ts` and wired as observers |
+| Media: caption family | `src/media/caption/*` | types, subtitles, utils (incl. `setChars`), position, seven presets, resolver. See semantic notes on sparse TextStyle |
+| Text rendering | `src/utils/text.ts` (full port) | tokenize/shape/render joined the Token seed; reads RenderSurface.ctx + Camera; no-ops without a surface. Gradient construction split into `src/systems/gradients.ts` (render system reuses it) |
+| Fonts | `src/fonts/{fixtures,utils}.ts` | WebFonts fixtures verbatim; `loadWebFont` uses the Fonts world trait, resolves the FontFaceSet via globalThis (worker-capable), guards on missing FontFace (headless), and drops db persistence (app wraps it). `getLocalFonts` + `restoreFonts` stay app-side |
+| Support utils | `src/utils/{color,async}.ts`, `clamp` in math | parseColor/colorToHex/rgbToColor (colord dep), AsyncMutex/attempt |
| Seeds (types only, rest follows) | `src/utils/text.ts` (Token/Line/TokenOptions), `src/capture/format.ts` (ContainerFormat) | The remaining engine files merge into these on their slice |
Verification style used so far: `npx tsc --noEmit` in the package, plus throwaway `smoke-test.tmp.ts` in the package root run with `npm exec --yes tsx@latest -- smoke-test.tmp.ts`, deleted afterwards (no test infra in the package yet). koota caps live worlds at 16; call `world.destroy()` in test loops.
@@ -60,7 +65,13 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All
- Not faithful on purpose: keyframe auto-sync on authored edits did not move into observers. koota change events don't say which field changed, and syncing every field of a trait would mint spurious keyframes on untouched properties. Editing surfaces call `syncKeyframeTrack` for the property they changed (resizeEntity already does for width/height).
- Size propagation is a Size observer now (`propagateSize` on add/change): bitecs restored sizes through `resizeEntity`, koota deserialization is a plain set. `resizeEntity` no longer calls propagateSize itself.
- `clampTrimToAssetDuration` now goes through `entity.set(Trim)` (fires change events) but only when actually clamping; bitecs wrote the store silently. The Trim onChange handler re-enters once through the clamp and no-ops on the second pass.
-- Still app-side from `api/observers.ts`: persistence scheduling, timeline index rebuilds, `syncInteractiveState`, Selected waveform show/hide, ClipHeight/Expanded, ExportSettings defaults, drag-origin traits. Media slice: dispose hooks on Deleted/Culled onAdd.
+- Still app-side from `api/observers.ts`: persistence scheduling, timeline index rebuilds, `syncInteractiveState`, Selected waveform show/hide, ClipHeight/Expanded, ExportSettings defaults, drag-origin traits.
+- Media handle traits renamed with a `Handle` suffix (`VideoDecoderHandle`, `AudioBusHandle`, ...) and typed to the concrete classes via type-only imports, so the trait and the class it stores (`AudioDecoder`, `HtmlHost`, ...) share one export surface without colliding. Resolvers are entity-based (`resolveVideoDecoder(world, entity)`), read Assets/Mode world traits, and store instances with add+set on the handle trait.
+- Dispose hooks are observers now: Deleted onAdd disposes decoders/hosts/buses across the tombstoned subtree, Culled onAdd disposes decoders, ChildOf add/remove disconnects audio buses, and `reactToAssetChange`'s Caption branch drops the caption decoder. The walkers (`src/media/dispose.ts`) deliberately recurse through Deleted descendants, like bitecs.
+- TextStyle became all-undefined defaults: a range override carries ONLY its explicitly set fields (sparse store slot = inherit from the node), which is what the bitecs set-observer's TextRange special case enforced. Node-level fallbacks (16px/Inter/400/leading 1) live in the text renderer's accessors, restoring the bitecs `?? 1` leading semantics the earlier trait defaults had drifted from.
+- Caption presets write playhead-driven text via `setChars` (raw store write, no change events) so per-frame caption content never reaches app persistence, same as the bitecs raw array writes. Preset fills/ranges go through createEntity/appendChild, so caches prime via the observers.
+- DOM caveats, all lazy (nothing touches DOM at import time): HtmlHost/SurfaceHost create elements at construction (html-in-canvas is Chromium-flag-only anyway), ShaderHost needs WebGPU, `loadWebFont` no-ops without FontFace. tsconfig gained `DOM.Iterable` + `@webgpu/types`.
+- `SequenceDecoder` iterates the structural `AssetDirectoryHandle` (entries with `getFile`) instead of concrete FileSystemFileHandle, so node hosts can feed frame directories.
- `switchActiveScene` only sets the ActiveScene world trait; timeline buffer clearing and index rebuild stay app-side. `frameSelection` calls it once (bitecs called it redundantly inside and outside the transaction).
- `insertAssetInTimeline` returns the created entity or null when no scene is active (no toast). Folder ops are sync and return the changed records (`deleteFolder` returns removed folder/asset ids) so the app can persist; `moveAssetsToFolder` updates records in place instead of re-fronting them like `saveAsset` did. Panel navigation state (current folder) stays app-side.
- Decoder/host handle traits (`src/traits/media.ts`) are AoS traits typed as opaque `object` until the decoders move in; `entity.get(VideoDecoder)` returns the instance directly.
@@ -79,17 +90,16 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All
## Remaining work (suggested order)
-1. **Media** (`src/media/`): move `engine/decoders/*` wholesale (video, audio, image, sequence, caption/, html, shader, surface, frame-cache, keyframe-index, time-stretcher, audio-peaks) plus `services/audio-bus.ts`. Then tighten the handle trait types in `src/traits/media.ts` to the concrete classes and delete the `Handle = object` placeholder. Reinstate the dispose hooks earlier slices dropped, as observers in `world/observers.ts`: Deleted onAdd → dispose decoders/hosts/bus, Culled onAdd → dispose decoders, ChildOf events → `disconnectAudioBus`, `reactToAssetChange`'s Caption branch → drop the caption decoder.
-2. **Playback system**: `systems/playback.ts` (computes `Computed.localTime/start/end/duration/visibility`, drives decoders, pushes into `FramePromises`, uses AudioEngine + Mounts). Depends on media.
-3. **Capture** (`src/capture/`): `engine/encode/*` (encoder, image-encoder, buffer, format, utils). Uses Mode, FramePromises, RenderSurface.
-4. **Render system**: `systems/render.ts` is runtime-capable (OffscreenCanvas) and needed for headless capture; it consumes text layout (`utils/text.ts` full port: layout/measure/render join the Token seed).
-5. **Fonts**: `engine/font/{utils,fixtures}.ts` (check DOM usage: document.fonts needs an injection seam or stays app-side).
-6. **Remaining utils**: captions, transition, contact-sheet, audio-sync, transcode (audit each for DOM/app deps; DOM-touching ones stay app-side).
-7. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.
-8. **CLI**: point node capture at the runtime package.
+1. **Playback system**: `systems/playback.ts` (computes `Computed.localTime/start/end/duration/visibility`, drives decoders via the media resolvers, pushes into `FramePromises`, uses AudioEngine + Mounts; `resolveAudioBus` lives here).
+2. **Capture** (`src/capture/`): `engine/encode/*` (encoder, image-encoder, buffer, format, utils). Uses Mode, FramePromises, RenderSurface.
+3. **Render system**: `systems/render.ts` is runtime-capable (OffscreenCanvas) and needed for headless capture; text rendering and gradients are already in (`utils/text.ts`, `systems/gradients.ts`; `addStopsTo` stayed private to gradients).
+4. **Remaining utils**: captions, transition, contact-sheet, audio-sync, transcode (audit each for DOM/app deps; DOM-touching ones stay app-side).
+5. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.
+6. **CLI**: point node capture at the runtime package.
## Layout reminders
- Package exports: root `.` plus `./traits`, `./systems`, `./media`, `./capture` (see package.json). Everything currently also re-exported from root `src/index.ts`.
- tsconfig sets `erasableSyntaxOnly: false` (enums with reverse lookups, same as apps/web).
-- Deps: koota (patched), mediabunny, animejs, sqids (asset/folder id allocation).
+- Deps: koota (patched), mediabunny, animejs, sqids (asset/folder id allocation), colord (parseColor). Dev: @webgpu/types (shader host).
+- AoS handle traits accept class instances and null through plain `entity.set(trait, value)`; `getStore` returns the value array.
diff --git a/packages/runtime/package.json b/packages/runtime/package.json
index 5149ac08..e036b811 100644
--- a/packages/runtime/package.json
+++ b/packages/runtime/package.json
@@ -22,11 +22,13 @@
],
"dependencies": {
"animejs": "^4.4.1",
+ "colord": "^2.9.3",
"koota": "^0.6.6",
"mediabunny": "^1.32.2",
"sqids": "^0.3.0"
},
"devDependencies": {
+ "@webgpu/types": "^0.1.71",
"typescript": "~5.9.3"
}
}
diff --git a/packages/runtime/src/actions/timing.ts b/packages/runtime/src/actions/timing.ts
index e993045c..512abe50 100644
--- a/packages/runtime/src/actions/timing.ts
+++ b/packages/runtime/src/actions/timing.ts
@@ -10,7 +10,7 @@ import { store } from '../world/store';
import { DEFAULT_DURATION_FRAMES } from '../constants';
import {
Geometry, Group, Paint, Audio, Caption, Cache, Computed, Delay,
- PlaybackRate, Trim,
+ PlaybackRate, Trim, CaptionDecoderHandle,
} from '../traits';
import { isGroupLike } from '../queries/predicates';
import { getParentNode } from '../queries/hierarchy';
@@ -139,8 +139,12 @@ export function reactToAssetChange(world: World, entity: Entity) {
} else if (entity.has(Audio)) {
reactToGeometryDurationChange(world, entity);
} else if (entity.has(Caption)) {
- // Re-pointed transcript: the caption decoder must re-resolve with the
- // new asset. Decoder disposal moves in with the media slice.
+ // Re-pointed transcript: drop the decoder so it re-resolves with the
+ // new asset.
+ if (entity.has(CaptionDecoderHandle)) {
+ entity.get(CaptionDecoderHandle)?.dispose();
+ entity.set(CaptionDecoderHandle, null);
+ }
}
}
diff --git a/packages/runtime/src/assets/types.ts b/packages/runtime/src/assets/types.ts
index 8a9c81af..ef3ac168 100644
--- a/packages/runtime/src/assets/types.ts
+++ b/packages/runtime/src/assets/types.ts
@@ -20,6 +20,8 @@ export interface AssetDirectoryHandle {
export type TranscriptWord = { text: string; start: number; end: number };
+export type WordGroup = TranscriptWord[];
+
export type Transcript = { text: string; words: TranscriptWord[] }[];
interface AssetBase {
diff --git a/packages/runtime/src/fonts/fixtures.ts b/packages/runtime/src/fonts/fixtures.ts
new file mode 100644
index 00000000..62f7b079
--- /dev/null
+++ b/packages/runtime/src/fonts/fixtures.ts
@@ -0,0 +1,62 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export const WebFonts = {
+ Inter: {
+ weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'],
+ url: 'https://fonts.gstatic.com/s/inter/v20/UcCo3FwrK3iLTcviYwYZ8UA3.woff2',
+ },
+ 'The Bold Font': {
+ weights: ['500'],
+ url: 'https://diffusion-studio-public.s3.eu-central-1.amazonaws.com/fonts/the-bold-font.ttf',
+ },
+ 'Komika Axis': {
+ weights: ['400'],
+ url: 'https://diffusion-studio-public.s3.eu-central-1.amazonaws.com/fonts/komika-axis.ttf',
+ },
+ Geologica: {
+ weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'],
+ url: 'https://fonts.gstatic.com/s/geologica/v1/oY1l8evIr7j9P3TN9YwNAdyjzUyDKkKdAGOJh1UlCDUIhAIdhCZOn1fLsig7jfvCCPHZckUWE1lELWNN-w.woff2',
+ },
+ Nunito: {
+ weights: ['200', '300', '400', '500', '600', '700', '800', '900'],
+ url: 'https://fonts.gstatic.com/s/nunito/v26/XRXV3I6Li01BKofINeaBTMnFcQ.woff2',
+ },
+ Figtree: {
+ weights: ['300', '400', '500', '600', '700', '800', '900'],
+ url: 'https://fonts.gstatic.com/s/figtree/v5/_Xms-HUzqDCFdgfMm4S9DaRvzig.woff2',
+ },
+ Urbanist: {
+ weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'],
+ url: 'https://fonts.gstatic.com/s/urbanist/v15/L0x-DF02iFML4hGCyMqlbS1miXK2.woff2',
+ },
+ Montserrat: {
+ weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'],
+ url: 'https://fonts.gstatic.com/s/montserrat/v26/JTUSjIg1_i6t8kCHKm459WlhyyTh89Y.woff2',
+ },
+ Bangers: {
+ weights: ['400'],
+ url: 'https://fonts.gstatic.com/s/bangers/v20/FeVQS0BTqb0h60ACH55Q2J5hm24.woff2',
+ },
+ Chewy: {
+ weights: ['400'],
+ url: 'https://fonts.gstatic.com/s/chewy/v18/uK_94ruUb-k-wn52KjI9OPec.woff2',
+ },
+ 'Source Code Pro': {
+ weights: ['200', '300', '400', '500', '600', '700', '800', '900'],
+ url: 'https://fonts.gstatic.com/s/sourcecodepro/v22/HI_SiYsKILxRpg3hIP6sJ7fM7PqlPevWnsUnxg.woff2',
+ },
+} as const;
+
+export const FONT_WEIGHTS = {
+ '100': 'Thin',
+ '200': 'Extra Light',
+ '300': 'Light',
+ '400': 'Normal',
+ '500': 'Medium',
+ '600': 'Semi Bold',
+ '700': 'Bold',
+ '800': 'Extra Bold',
+ '900': 'Black',
+} as const;
diff --git a/packages/runtime/src/fonts/index.ts b/packages/runtime/src/fonts/index.ts
index a8e32130..38e756ee 100644
--- a/packages/runtime/src/fonts/index.ts
+++ b/packages/runtime/src/fonts/index.ts
@@ -1,2 +1,3 @@
-// Font loading; fixtures and utils follow from engine/font/.
export * from './types';
+export * from './fixtures';
+export * from './utils';
diff --git a/packages/runtime/src/fonts/utils.ts b/packages/runtime/src/fonts/utils.ts
new file mode 100644
index 00000000..64d3b4a7
--- /dev/null
+++ b/packages/runtime/src/fonts/utils.ts
@@ -0,0 +1,97 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Web font loading (was engine/font/utils.ts, minus the app-side pieces:
+// getLocalFonts needs window.queryLocalFonts, and font persistence is the
+// app's IndexedDB concern; it wraps loadWebFont and re-calls it on restore).
+
+import { WebFonts } from './fixtures';
+import { FontStyle } from '../constants';
+import { Fonts } from '../traits';
+
+import type { World } from 'koota';
+import type * as types from './types';
+
+const STYLE_MAP = {
+ [FontStyle.NORMAL]: 'normal',
+ [FontStyle.ITALIC]: 'italic',
+ [FontStyle.OBLIQUE]: 'oblique',
+} as const;
+
+/** The environment's FontFaceSet: document.fonts on a page, self.fonts in a worker. */
+function getFontFaceSet(): FontFaceSet | null {
+ const scope = globalThis as { document?: { fonts?: FontFaceSet }; fonts?: FontFaceSet };
+ return scope.document?.fonts ?? scope.fonts ?? null;
+}
+
+/**
+ * Get common web fonts
+ */
+export function getWebFonts(): types.FontSources[] {
+ return Object.keys(WebFonts).map((family) => {
+ return {
+ family,
+ variants: WebFonts[family as keyof typeof WebFonts].weights.map((weight) => {
+ return {
+ family,
+ source: `url(${WebFonts[family as keyof typeof WebFonts].url})`,
+ weight: weight,
+ };
+ }),
+ };
+ });
+}
+
+export async function loadWebFont(
+ world: World,
+ family: keyof typeof WebFonts,
+ style: FontStyle = FontStyle.NORMAL,
+ weight?: string,
+): Promise<types.FontSource> {
+ const source = `url(${WebFonts[family].url})`;
+ const font: types.FontSource = {
+ source,
+ family,
+ weight,
+ style,
+ };
+
+ const fonts = world.get(Fonts)?.list;
+ // No FontFace = headless environment without font rasterization; text
+ // measurement falls back to the system font.
+ if (!fonts || typeof FontFace === 'undefined') return font;
+
+ // Already loaded
+ if (fonts.some((f) => f.source === font.source && f.weight === font.weight)) {
+ return font;
+ }
+
+ fonts.push(font);
+
+ // Load all weights if no weight is specified
+ if (!weight) {
+ const config = WebFonts[family as keyof typeof WebFonts];
+ const weights = config.weights;
+ weight = weights.length > 1 ? `${weights[0]} ${weights[weights.length - 1]}` : weights[0];
+ }
+
+ const fontFace = new FontFace(family, source, { weight, style: STYLE_MAP[style] });
+
+ await new Promise((resolve, reject) => {
+ fontFace
+ .load()
+ .then((loaded) => {
+ getFontFaceSet()?.add(loaded);
+ resolve(null);
+ })
+ .catch((error) => {
+ world.set(Fonts, {
+ list: fonts.filter((f) => f.source !== font.source && f.weight !== font.weight),
+ });
+ reject(error);
+ });
+ });
+
+ return font;
+}
diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts
index 0c7d0123..7c8faa44 100644
--- a/packages/runtime/src/index.ts
+++ b/packages/runtime/src/index.ts
@@ -9,6 +9,7 @@ export * from './actions';
export * from './assets/types';
export * from './systems';
export * from './math';
+export * from './media';
export * from './capture';
export * from './fonts';
export * from './utils';
diff --git a/packages/runtime/src/math/common.ts b/packages/runtime/src/math/common.ts
index 3f13b64f..1b544b55 100644
--- a/packages/runtime/src/math/common.ts
+++ b/packages/runtime/src/math/common.ts
@@ -16,3 +16,7 @@ export function degToRad(deg: number): number {
export function radToDeg(rad: number): number {
return rad * RAD_TO_DEG;
}
+
+export function clamp(value: number, min: number, max: number): number {
+ return Math.min(Math.max(value, min), max);
+}
diff --git a/packages/runtime/src/media/audio-bus.ts b/packages/runtime/src/media/audio-bus.ts
new file mode 100644
index 00000000..85d2d1bb
--- /dev/null
+++ b/packages/runtime/src/media/audio-bus.ts
@@ -0,0 +1,71 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { store } from '../world/store';
+import { Muted, Computed, AudioEngine } from '../traits';
+import { attempt } from '../utils/async';
+import { assert } from '../utils/assert';
+
+import type { Entity, World } from 'koota';
+
+/**
+ * Per-entity audio bus. Each clip and each scene gets its own gain node.
+ * Child buses connect directly into the parent bus's input; there is no
+ * per-track sub-mix anymore (each clip is its own layer).
+ */
+export class AudioBus {
+ public context: BaseAudioContext;
+
+ private gain: GainNode;
+ private entity: Entity;
+ private world: World;
+ private _input: AudioNode;
+
+ public constructor(world: World, entity: Entity) {
+ const context = world.get(AudioEngine)?.context;
+ assert(context, 'World has no audio context');
+ this.context = context;
+ this.gain = context.createGain();
+ this._input = this.gain;
+ this.entity = entity;
+ this.world = world;
+ }
+
+ public get input(): AudioNode {
+ return this._input;
+ }
+
+ public getGain(): GainNode {
+ return this.gain;
+ }
+
+ public sync(): void {
+ this.gain.gain.value = this.getVolume();
+ }
+
+ public connect(node: AudioNode) {
+ this.gain.connect(node);
+ }
+
+ public mute(): void {
+ this.gain.gain.value = 0;
+ }
+
+ public disconnect() {
+ attempt(() => this.gain.disconnect());
+ this._input = this.gain;
+ }
+
+ private getVolume(): number {
+ const volumeDb = store(this.world, Computed).volume[this.entity.id()] ?? 0;
+ const muted = this.entity.has(Muted);
+
+ /** Minimum dB value: treated as silence (maps to linear gain 0). */
+ if (muted || volumeDb === -Infinity) {
+ return 0;
+ }
+
+ return Math.pow(10, volumeDb / 20);
+ }
+}
diff --git a/packages/runtime/src/media/audio-peaks.ts b/packages/runtime/src/media/audio-peaks.ts
new file mode 100644
index 00000000..71450578
--- /dev/null
+++ b/packages/runtime/src/media/audio-peaks.ts
@@ -0,0 +1,129 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { AudioSampleSink } from 'mediabunny';
+
+import { getAudioTrack } from './audio';
+
+import type { AudioAsset, VideoAsset } from '../assets/types';
+
+const peakCache = new Map<string, Uint8ClampedArray>();
+const inflight = new Map<string, Promise<Uint8ClampedArray | null>>();
+
+const TARGET_BARS = 128;
+
+export function clearAudioPeaksCache() {
+ peakCache.clear();
+ inflight.clear();
+}
+
+/**
+ * Synchronous, allocation-free peak lookup for hot paths (render loop).
+ * Returns cached peaks or `null`. Kicks off a build on first call.
+ */
+export function getAudioPeaks(
+ asset: AudioAsset | VideoAsset,
+): Uint8ClampedArray | null {
+ const key = asset.id;
+
+ const cached = peakCache.get(key);
+ if (cached) return cached;
+
+ if (!inflight.has(key)) {
+ inflight.set(key, buildPeaks(key, asset));
+ }
+
+ return null;
+}
+
+/**
+ * Async peak lookup for UI components that can await.
+ * Returns peaks from cache or waits for the build to finish.
+ */
+export async function getAudioPeaksAsync(
+ asset: AudioAsset | VideoAsset,
+): Promise<Uint8ClampedArray | null> {
+ const key = asset.id;
+
+ const cached = peakCache.get(key);
+ if (cached) return cached;
+
+ let promise = inflight.get(key);
+ if (!promise) {
+ promise = buildPeaks(key, asset);
+ inflight.set(key, promise);
+ }
+
+ return promise;
+}
+
+async function buildPeaks(
+ key: string,
+ asset: AudioAsset | VideoAsset,
+): Promise<Uint8ClampedArray | null> {
+ try {
+ const track = await getAudioTrack(asset);
+ if (!track) return null;
+
+ const sink = new AudioSampleSink(track);
+ const chunks: Uint8ClampedArray[] = [];
+
+ for await (const sample of sink.samples()) {
+ let floats: Float32Array;
+ let numChannels: number;
+ let numFrames: number;
+ try {
+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });
+ floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);
+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });
+
+ numChannels = sample.numberOfChannels;
+ numFrames = floats.length / numChannels;
+ } finally {
+ sample.close();
+ }
+
+ const framePeaks = new Uint8ClampedArray(numFrames);
+ for (let f = 0; f < numFrames; f++) {
+ let max = 0;
+ for (let ch = 0; ch < numChannels; ch++) {
+ max = Math.max(max, Math.pow(Math.abs(floats[f * numChannels + ch]), 0.8));
+ }
+ framePeaks[f] = Math.floor(max * 255);
+ }
+ chunks.push(framePeaks);
+ }
+
+ if (chunks.length === 0) return null;
+
+ const totalFrames = chunks.reduce((s, c) => s + c.length, 0);
+ const merged = new Uint8ClampedArray(totalFrames);
+ let offset = 0;
+ for (const chunk of chunks) {
+ merged.set(chunk, offset);
+ offset += chunk.length;
+ }
+
+ const peaks = new Uint8ClampedArray(TARGET_BARS);
+ const ratio = totalFrames / TARGET_BARS;
+
+ for (let i = 0; i < TARGET_BARS; i++) {
+ const start = Math.floor(i * ratio);
+ const end = Math.floor((i + 1) * ratio);
+ let max = 0;
+ for (let j = start; j < end && j < totalFrames; j++) {
+ max = Math.max(max, merged[j]);
+ }
+ peaks[i] = max;
+ }
+
+ peakCache.set(key, peaks);
+ return peaks;
+ } catch (e) {
+ console.error('Failed to build audio peaks', e);
+ return null;
+ } finally {
+ inflight.delete(key);
+ }
+}
diff --git a/packages/runtime/src/media/audio.ts b/packages/runtime/src/media/audio.ts
new file mode 100644
index 00000000..faedff1f
--- /dev/null
+++ b/packages/runtime/src/media/audio.ts
@@ -0,0 +1,277 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Input, ALL_FORMATS, BlobSource, AudioBufferSink, type WrappedAudioBuffer, type InputAudioTrack } from 'mediabunny';
+
+import { AssetId, AudioDecoderHandle, Assets } from '../traits';
+import { AsyncMutex } from '../utils/async';
+import { assert } from '../utils/assert';
+import { getAssetFile } from '../actions/assets';
+import { TimeStretcher } from './time-stretcher';
+
+import type { Entity, World } from 'koota';
+import type { AudioAsset, VideoAsset } from '../assets/types';
+import type { AudioBus } from './audio-bus';
+
+const audioTrackCache = new Map<string, Promise<InputAudioTrack | null>>();
+
+export function clearAudioTrackCache() {
+ audioTrackCache.clear();
+}
+
+/**
+ * Keyed by asset ID so multiple decoders for the same source
+ * audio reuse a single Input + InputAudioTrack.
+ */
+export function getAudioTrack(source: AudioAsset | VideoAsset) {
+ let promise = audioTrackCache.get(source.id);
+ if (promise) return promise;
+
+ promise = (async () => {
+ try {
+ const blob = await getAssetFile(source);
+ const input = new Input({
+ formats: ALL_FORMATS,
+ source: new BlobSource(blob),
+ });
+ return await input.getPrimaryAudioTrack();
+ } catch {
+ audioTrackCache.delete(source.id);
+ return null;
+ }
+ })();
+
+ audioTrackCache.set(source.id, promise);
+ return promise;
+}
+
+type PlayOptions = {
+ relativeFrom: number;
+ relativeTo: number;
+ trimStart: number;
+ trimEnd: number;
+ playbackRate: number;
+ currentTime: number;
+ relativeDelay: number;
+}
+
+export class AudioDecoder {
+ readonly assetId: string;
+
+ private readonly asset: AudioAsset | VideoAsset;
+ private readonly mutex = new AsyncMutex();
+ private readonly audioNodes = new Set<AudioBufferSourceNode>();
+
+ private iterator: AsyncGenerator<WrappedAudioBuffer, void, unknown> | null = null;
+ private firstBuffer: WrappedAudioBuffer | null = null;
+ private lastBuffer: WrappedAudioBuffer | null = null;
+ private stretcher: TimeStretcher | null = null;
+ private nextTimestamp = 0;
+ private audioTrack: InputAudioTrack | null = null;
+ private sink!: AudioBufferSink;
+
+ public ready = false;
+
+ public constructor(asset: AudioAsset | VideoAsset) {
+ this.asset = asset;
+ this.assetId = asset.id;
+ }
+
+ public async init() {
+ this.audioTrack = await getAudioTrack(this.asset);
+ assert(this.audioTrack, 'Audio track not found');
+ this.sink = new AudioBufferSink(this.audioTrack);
+ this.ready = true;
+ }
+
+ private renderData(bus: AudioBus, options: PlayOptions, data: Float32Array[] | null, sampleRate: number) {
+ if (!data) {
+ return;
+ }
+
+ const { relativeDelay, playbackRate } = options;
+
+ const trimStart = options.trimStart / playbackRate;
+ const trimEnd = options.trimEnd / playbackRate;
+ const context = bus.context;
+ const input = bus.input;
+
+ const newBuffer = new AudioBuffer({
+ length: data[0]!.length,
+ sampleRate,
+ numberOfChannels: data.length,
+ });
+ for (let c = 0; c < data.length; c++) {
+ newBuffer.copyToChannel(data[c] as Float32Array<ArrayBuffer>, c);
+ }
+
+ const newBufferTimestamp = this.nextTimestamp;
+ this.nextTimestamp += newBuffer.duration;
+
+ let start = relativeDelay + newBufferTimestamp;
+ let offset = 0;
+ let duration = newBuffer.duration;
+
+ // Check if the buffer goes beyond the clip's end, and if so, truncate it
+ if (newBufferTimestamp + duration > trimEnd) {
+ duration = trimEnd - newBufferTimestamp;
+ }
+
+ // Check if the buffer starts before the clip's start, and if so, trim the start
+ if (newBufferTimestamp < trimStart) {
+ start += trimStart - newBufferTimestamp;
+ offset += trimStart - newBufferTimestamp;
+ }
+
+ // Check if the start is before the context's current time (which is illegal)
+ if (start < context.currentTime) {
+ offset += context.currentTime - start;
+ start = context.currentTime;
+ }
+
+ // Dumb, DIRTY hack. For some reason (probably due to some bug in the Web Audio API), sometimes a clicking noise
+ // can be heard on the boundary between two audio clips. By shifting all audio buffers by half a sample, this is
+ // typically avoided. My guess is that it forces the renderer into a sub-sample rendering algorithm which doesn't
+ // have the buggy behavior.
+ start += 0.5 / context.sampleRate;
+
+ duration -= offset;
+ duration = Math.max(0, duration);
+
+ const bufferSource = context.createBufferSource();
+ bufferSource.buffer = newBuffer;
+ bufferSource.connect(input);
+ bufferSource.start(start, offset, duration);
+
+ this.audioNodes.add(bufferSource);
+ bufferSource.onended = () => this.audioNodes.delete(bufferSource);
+ }
+
+ public async playTo(bus: AudioBus | null, options: PlayOptions) {
+ if (!this.asset.channels || !this.asset.sampleRate || !bus) {
+ // Asset has no audio data
+ return;
+ }
+
+ const { relativeFrom, relativeTo } = options;
+
+ const release = await this.mutex.acquire();
+
+ const firstTs = this.firstBuffer?.timestamp ?? Number.POSITIVE_INFINITY;
+ const lastTs = this.lastBuffer?.timestamp ?? Number.NEGATIVE_INFINITY;
+ const isBufferOutOfRange = relativeFrom < firstTs || lastTs < relativeFrom - 1;
+
+ try {
+ if (this.iterator == null || this.stretcher == null || isBufferOutOfRange) {
+ // Close the previous iterator so its pre-decoded AudioSamples get released.
+ await this.iterator?.return();
+ this.iterator = this.sink.buffers(relativeFrom);
+ this.firstBuffer = null;
+ this.lastBuffer = null;
+ this.stretcher = null;
+ }
+
+ while (true) {
+ const nextBuffer = (await this.iterator.next()).value;
+ // Use the actual decoded sample rate if available
+ const sampleRate = nextBuffer?.buffer.sampleRate ?? this.asset.sampleRate;
+ const numberOfChannels = nextBuffer?.buffer.numberOfChannels ?? this.asset.channels;
+ const playbackRate = 1 / options.playbackRate;
+
+ this.stretcher ??= new TimeStretcher(numberOfChannels, sampleRate, playbackRate);
+
+ if (!nextBuffer) {
+ this.renderData(bus, options, this.stretcher.finalize(), sampleRate);
+ break;
+ }
+
+ if (!this.firstBuffer) {
+ this.nextTimestamp = nextBuffer.timestamp / options.playbackRate;
+ }
+
+ if (this.lastBuffer) {
+ const sampleGap = Math.round(
+ nextBuffer.timestamp - (this.lastBuffer.timestamp + this.lastBuffer.duration)
+ );
+
+ if (sampleGap > 0) {
+ this.renderData(
+ bus,
+ options,
+ Array(numberOfChannels).fill(new Float32Array(sampleGap)),
+ sampleRate
+ );
+ }
+ }
+
+ this.firstBuffer ??= nextBuffer;
+ this.lastBuffer = nextBuffer;
+
+ const buffers: Float32Array[] = [];
+ for (let c = 0; c < nextBuffer.buffer.numberOfChannels; c++) {
+ const channelData = nextBuffer.buffer.getChannelData(c);
+ buffers.push(channelData);
+ }
+
+ // TODO: Handle gap in timestamps case
+
+ this.renderData(bus, options, this.stretcher.append(buffers), sampleRate);
+
+ if (nextBuffer.timestamp >= relativeTo) {
+ break;
+ }
+ }
+ } finally {
+ release();
+ }
+ }
+
+ public reset() {
+ if (!this.iterator) return;
+
+ this.iterator?.return();
+ this.iterator = null;
+ this.firstBuffer = null;
+ this.lastBuffer = null;
+
+ for (const node of this.audioNodes) {
+ node.stop();
+ }
+ this.audioNodes.clear();
+ }
+}
+
+
+type ResolvedAudioDecoder = {
+ decoder: AudioDecoder;
+ initPromise: Promise<void> | null;
+};
+
+export function resolveAudioDecoder(world: World, entity: Entity): ResolvedAudioDecoder | null {
+ const assetId = entity.get(AssetId)?.value;
+ if (!assetId) return null;
+
+ const existing = entity.get(AudioDecoderHandle);
+ if (existing && existing.assetId === assetId) {
+ return {
+ decoder: existing,
+ initPromise: null,
+ };
+ }
+
+ // Asset changed: reset old decoder and create a new one.
+ if (existing) existing.reset();
+
+ const asset = world.get(Assets)?.get(assetId);
+ if (!asset || (asset.type !== 'AUDIO' && asset.type !== 'VIDEO')) return null;
+ if (asset.type === 'VIDEO' && !asset.channels) return null;
+
+ const decoder = new AudioDecoder(asset);
+ entity.add(AudioDecoderHandle);
+ entity.set(AudioDecoderHandle, decoder);
+ return {
+ decoder,
+ initPromise: decoder.init(),
+ };
+}
diff --git a/packages/runtime/src/media/caption/cascade.ts b/packages/runtime/src/media/caption/cascade.ts
new file mode 100644
index 00000000..33d9ab37
--- /dev/null
+++ b/packages/runtime/src/media/caption/cascade.ts
@@ -0,0 +1,101 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';
+import { Paint, Color, TextStyle } from '../../traits';
+import { renderText } from '../../utils/text';
+import { loadWebFont } from '../../fonts/utils';
+import { groupBy, findActiveGroup, resolveTranscript, setChars } from './utils';
+import { placeCaption } from './position';
+import { createEntity } from '../../actions/entities';
+import { appendChild } from '../../actions/hierarchy';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '../../assets/types';
+import type { CaptionDecoder } from './types';
+
+const WIDTH = 800;
+const HEIGHT = 200;
+
+export class CascadeCaptionDecoder implements CaptionDecoder {
+ public readonly type = CaptionType.CASCADE;
+ public groups: ReturnType<typeof groupBy> = [];
+ public ready = false;
+
+ private readonly asset: Asset;
+ private currentText = '';
+
+ constructor(asset: Asset) {
+ this.asset = asset;
+ this.init();
+ }
+
+ private async init() {
+ if (this.ready) return;
+ const transcript = await resolveTranscript(this.asset);
+ this.groups = groupBy(transcript, { length: 50 });
+ this.ready = true;
+ }
+
+ public reposition(world: World, entity: Entity): boolean {
+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, x: 100, defaultAlign: CaptionAlign.BOTTOM });
+ }
+
+ public applyStyles(world: World, entity: Entity): void {
+ if (!this.reposition(world, entity)) return;
+
+ entity.add(TextStyle);
+ entity.set(TextStyle, {
+ fontFamily: 'Inter',
+ fontWeight: '300',
+ fontSize: 50,
+ textAlign: TextAlign.LEFT,
+ textBaseline: TextBaseline.TOP,
+ fontStyle: FontStyle.NORMAL,
+ textCase: TextCase.ORIGINAL,
+ leading: 1.2,
+ });
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: 0xFFFFFF });
+ appendChild(world, fill, entity);
+
+ loadWebFont(world, 'Inter');
+ }
+
+ public seekTo(world: World, entity: Entity, relativeTime: number): void {
+ const groupIndex = findActiveGroup(this.groups, relativeTime);
+
+ if (groupIndex === -1) {
+ setChars(world, entity, '');
+ this.currentText = '';
+ return;
+ }
+
+ const group = this.groups[groupIndex]!;
+
+ // Progressive reveal: only show words that have started
+ const text = group
+ .filter(word => word.start <= relativeTime)
+ .map(w => w.text)
+ .join(' ');
+
+ if (text !== this.currentText) {
+ this.currentText = text;
+ setChars(world, entity, text);
+ }
+ }
+
+ public draw(world: World, entity: Entity): void {
+ renderText(world, entity);
+ }
+
+ public dispose(): void {
+ this.groups = [];
+ this.currentText = '';
+ }
+}
diff --git a/packages/runtime/src/media/caption/classic.ts b/packages/runtime/src/media/caption/classic.ts
new file mode 100644
index 00000000..12d5fd12
--- /dev/null
+++ b/packages/runtime/src/media/caption/classic.ts
@@ -0,0 +1,111 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';
+import { Paint, Color, Shadow, Appearance, Blur, Offset, TextStyle } from '../../traits';
+import { renderText } from '../../utils/text';
+import { loadWebFont } from '../../fonts/utils';
+import { groupBy, findActiveGroup, resolveTranscript, setChars } from './utils';
+import { placeCaption } from './position';
+import { createEntity } from '../../actions/entities';
+import { appendChild } from '../../actions/hierarchy';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '../../assets/types';
+import type { CaptionDecoder } from './types';
+
+export const CLASSIC_PRESET_WIDTH = 600;
+export const CLASSIC_PRESET_HEIGHT = 100;
+
+export class ClassicCaptionDecoder implements CaptionDecoder {
+ public readonly type = CaptionType.CLASSIC;
+ public groups: ReturnType<typeof groupBy> = [];
+ public ready = false;
+
+ private readonly asset: Asset;
+ private currentGroupIndex = -1;
+
+ constructor(asset: Asset) {
+ this.asset = asset;
+ this.init();
+ }
+
+ private async init() {
+ if (this.ready) return;
+ const transcript = await resolveTranscript(this.asset);
+ this.groups = groupBy(transcript, { duration: 0.2 });
+ this.ready = true;
+ }
+
+ public reposition(world: World, entity: Entity): boolean {
+ return placeCaption(world, entity, {
+ width: CLASSIC_PRESET_WIDTH,
+ height: CLASSIC_PRESET_HEIGHT,
+ defaultAlign: CaptionAlign.CENTER,
+ });
+ }
+
+ public applyStyles(world: World, entity: Entity): void {
+ if (!this.reposition(world, entity)) return;
+
+ entity.add(TextStyle);
+ entity.set(TextStyle, {
+ fontFamily: 'Urbanist',
+ fontWeight: '600',
+ fontSize: 62,
+ textAlign: TextAlign.CENTER,
+ textBaseline: TextBaseline.MIDDLE,
+ textCase: TextCase.LOWER,
+ fontStyle: FontStyle.NORMAL,
+ leading: 1,
+ });
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: 0xFFFFFF });
+ appendChild(world, fill, entity);
+
+ const shadow = createEntity(world);
+ shadow.add(Shadow);
+ shadow.add(Color);
+ shadow.set(Color, { value: 0x000000 });
+ shadow.add(Appearance);
+ shadow.set(Appearance, { opacity: 1 });
+ shadow.add(Blur);
+ shadow.set(Blur, { value: 28 });
+ shadow.add(Offset);
+ shadow.set(Offset, { x: 0, y: 5 });
+ appendChild(world, shadow, entity);
+
+ loadWebFont(world, 'Urbanist');
+ }
+
+ public seekTo(world: World, entity: Entity, relativeTime: number): void {
+ const groupIndex = findActiveGroup(this.groups, relativeTime);
+
+ if (groupIndex === -1) {
+ setChars(world, entity, '');
+ this.currentGroupIndex = -1;
+ return;
+ }
+
+ const group = this.groups[groupIndex]!;
+
+ if (groupIndex !== this.currentGroupIndex) {
+ this.currentGroupIndex = groupIndex;
+ setChars(world, entity, group.map(w => w.text).join(' '));
+ }
+ }
+
+ public draw(world: World, entity: Entity): void {
+ renderText(world, entity);
+ }
+
+ public dispose(): void {
+ this.groups = [];
+ this.currentGroupIndex = -1;
+ }
+}
diff --git a/packages/runtime/src/media/caption/guinea.ts b/packages/runtime/src/media/caption/guinea.ts
new file mode 100644
index 00000000..69c3504d
--- /dev/null
+++ b/packages/runtime/src/media/caption/guinea.ts
@@ -0,0 +1,155 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { store } from '../../world/store';
+import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';
+import { Paint, Color, Caption, ItemIndex, TextRange, TextStyle } from '../../traits';
+import { renderText } from '../../utils/text';
+import { loadWebFont } from '../../fonts/utils';
+import { groupBy, findActiveGroup, splitSequence, clearTextRanges, resolveTranscript, setChars } from './utils';
+import { placeCaption } from './position';
+import { createEntity } from '../../actions/entities';
+import { appendChild } from '../../actions/hierarchy';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '../../assets/types';
+import type { CaptionDecoder } from './types';
+
+const WIDTH = 700;
+const HEIGHT = 200;
+
+const HIGHLIGHT_COLOR_0 = 0xF55353;
+const HIGHLIGHT_COLOR_1 = 0xFEB139;
+const HIGHLIGHT_COLOR_2 = 0xF6F54D;
+
+const PSEUDO_RANDOM_SEQUENCE = [0, 1, 0, 2, 1, 1, 0, 0, 1, 0, 2, 1, 0, 2, 1, 0, 2, 1];
+
+export class GuineaCaptionDecoder implements CaptionDecoder {
+ public readonly type = CaptionType.GUINEA;
+ public groups: ReturnType<typeof groupBy> = [];
+ public ready = false;
+
+ private readonly asset: Asset;
+ private currentGroupIndex = -1;
+ private activeSplit: 'left' | 'right' | null = null;
+ private fill: Entity | null = null;
+ private colorIndex = 0;
+
+ constructor(asset: Asset) {
+ this.asset = asset;
+ this.init();
+ }
+
+ private async init() {
+ if (this.ready) return;
+ const transcript = await resolveTranscript(this.asset);
+ this.groups = groupBy(transcript, { length: 18 });
+ this.ready = true;
+ }
+
+ public reposition(world: World, entity: Entity): boolean {
+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.CENTER });
+ }
+
+ public applyStyles(world: World, entity: Entity): void {
+ if (!this.reposition(world, entity)) return;
+
+ entity.add(TextStyle);
+ entity.set(TextStyle, {
+ fontFamily: 'The Bold Font',
+ fontWeight: '500',
+ fontSize: 62,
+ textAlign: TextAlign.CENTER,
+ textBaseline: TextBaseline.MIDDLE,
+ textCase: TextCase.UPPER,
+ fontStyle: FontStyle.NORMAL,
+ leading: 1,
+ });
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: 0xFFFFFF });
+ appendChild(world, fill, entity);
+
+ loadWebFont(world, 'The Bold Font');
+ }
+
+ public seekTo(world: World, entity: Entity, relativeTime: number): void {
+ const groupIndex = findActiveGroup(this.groups, relativeTime);
+
+ if (groupIndex === -1) {
+ setChars(world, entity, '');
+ clearTextRanges(world, entity);
+ this.currentGroupIndex = -1;
+ this.activeSplit = null;
+ return;
+ }
+
+ const group = this.groups[groupIndex]!;
+ const [left, right] = splitSequence(group);
+ const leftText = left.map(w => w.text).join(' ');
+ const rightText = right.map(w => w.text).join(' ');
+ const text = rightText ? `${leftText}\n${rightText}` : leftText;
+ const activeSplit = right.length > 0 && relativeTime >= right[0]!.start ? 'right' : 'left';
+
+ if (groupIndex !== this.currentGroupIndex || activeSplit !== this.activeSplit) {
+ this.currentGroupIndex = groupIndex;
+ this.activeSplit = activeSplit;
+ setChars(world, entity, text);
+
+ // Active line gets a random highlight color and a slightly larger font (1.1x).
+ clearTextRanges(world, entity);
+ const start = activeSplit === 'left' ? 0 : leftText.length + 1;
+ const end = activeSplit === 'left' ? leftText.length : text.length;
+ if (end > start) {
+ const baseSize = store(world, TextStyle).fontSize[entity.id()] ?? 62;
+ const fontSize = Math.round(baseSize * 1.1);
+
+ const range = createEntity(world);
+ range.add(TextRange);
+ range.set(TextRange, { start, end });
+ range.add(TextStyle);
+ range.set(TextStyle, { fontSize });
+ appendChild(world, range, entity);
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: HIGHLIGHT_COLOR_0 });
+ fill.add(ItemIndex);
+ fill.set(ItemIndex, { value: 0 });
+ appendChild(world, fill, range);
+ this.fill = fill;
+
+ this.colorIndex++;
+ }
+ }
+ }
+
+ public draw(world: World, entity: Entity): void {
+ const authored = entity.get(Caption)?.colors;
+ const colors = [
+ authored?.[0] ?? HIGHLIGHT_COLOR_0,
+ authored?.[1] ?? HIGHLIGHT_COLOR_1,
+ authored?.[2] ?? HIGHLIGHT_COLOR_2,
+ ] as const;
+
+ const index = PSEUDO_RANDOM_SEQUENCE[this.colorIndex % PSEUDO_RANDOM_SEQUENCE.length]!;
+
+ if (this.fill) {
+ store(world, Color).value[this.fill.id()] = colors[index]!;
+ }
+
+ renderText(world, entity);
+ }
+
+ public dispose(): void {
+ this.groups = [];
+ this.currentGroupIndex = -1;
+ this.activeSplit = null;
+ }
+}
diff --git a/packages/runtime/src/media/caption/index.ts b/packages/runtime/src/media/caption/index.ts
new file mode 100644
index 00000000..457608ca
--- /dev/null
+++ b/packages/runtime/src/media/caption/index.ts
@@ -0,0 +1,81 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { CaptionType } from '../../constants';
+import { AssetId, Caption, TextStyle, CaptionDecoderHandle, Assets } from '../../traits';
+import { ClassicCaptionDecoder } from './classic';
+import { CascadeCaptionDecoder } from './cascade';
+import { SpotlightCaptionDecoder } from './spotlight';
+import { WhisperCaptionDecoder } from './whisper';
+import { PaperCaptionDecoder } from './paper';
+import { GuineaCaptionDecoder } from './guinea';
+import { StarkCaptionDecoder } from './stark';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '../../assets/types';
+import type { CaptionDecoder } from './types';
+
+export type { CaptionDecoder } from './types';
+export { ClassicCaptionDecoder, CLASSIC_PRESET_WIDTH, CLASSIC_PRESET_HEIGHT } from './classic';
+export { CascadeCaptionDecoder } from './cascade';
+export { SpotlightCaptionDecoder } from './spotlight';
+export { WhisperCaptionDecoder } from './whisper';
+export { PaperCaptionDecoder } from './paper';
+export { GuineaCaptionDecoder } from './guinea';
+export { StarkCaptionDecoder } from './stark';
+export * from './position';
+export * from './subtitles';
+export * from './utils';
+
+function createCaptionDecoder(type: CaptionType, asset: Asset): CaptionDecoder {
+ switch (type) {
+ case CaptionType.CLASSIC:
+ return new ClassicCaptionDecoder(asset);
+ case CaptionType.CASCADE:
+ return new CascadeCaptionDecoder(asset);
+ case CaptionType.SPOTLIGHT:
+ return new SpotlightCaptionDecoder(asset);
+ case CaptionType.WHISPER:
+ return new WhisperCaptionDecoder(asset);
+ case CaptionType.PAPER:
+ return new PaperCaptionDecoder(asset);
+ case CaptionType.GUINEA:
+ return new GuineaCaptionDecoder(asset);
+ case CaptionType.STARK:
+ return new StarkCaptionDecoder(asset);
+ default:
+ return new ClassicCaptionDecoder(asset);
+ }
+}
+
+/**
+ * Lazily resolve (or create) a caption decoder for a caption entity.
+ * Recreates the decoder when the caption type changes.
+ * Returns null if the transcript asset isn't available yet.
+ */
+export function resolveCaptionDecoder(world: World, entity: Entity): CaptionDecoder | null {
+ const assetId = entity.get(AssetId)?.value;
+ if (!assetId) return null;
+
+ const captionType = entity.get(Caption)?.type ?? CaptionType.CLASSIC;
+ const existing = entity.get(CaptionDecoderHandle);
+
+ if (existing && captionType === existing.type) return existing;
+
+ const typeChanged = existing != null;
+ existing?.dispose();
+
+ const asset = world.get(Assets)?.get(assetId);
+ if (!asset) return null;
+
+ const decoder = createCaptionDecoder(captionType, asset);
+ entity.add(CaptionDecoderHandle);
+ entity.set(CaptionDecoderHandle, decoder);
+
+ if (typeChanged || !entity.has(TextStyle)) {
+ decoder.applyStyles(world, entity);
+ }
+
+ return decoder;
+}
diff --git a/packages/runtime/src/media/caption/paper.ts b/packages/runtime/src/media/caption/paper.ts
new file mode 100644
index 00000000..cedceb5b
--- /dev/null
+++ b/packages/runtime/src/media/caption/paper.ts
@@ -0,0 +1,119 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';
+import { Paint, Color, TextRange, TextStyle } from '../../traits';
+import { renderText } from '../../utils/text';
+import { loadWebFont } from '../../fonts/utils';
+import { groupBy, findActiveGroup, splitSequence, clearTextRanges, resolveTranscript, setChars } from './utils';
+import { placeCaption } from './position';
+import { createEntity } from '../../actions/entities';
+import { appendChild } from '../../actions/hierarchy';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '../../assets/types';
+import type { CaptionDecoder } from './types';
+
+const WIDTH = 700;
+const HEIGHT = 200;
+
+export class PaperCaptionDecoder implements CaptionDecoder {
+ public readonly type = CaptionType.PAPER;
+ public groups: ReturnType<typeof groupBy> = [];
+ public ready = false;
+
+ private readonly asset: Asset;
+ private currentGroupIndex = -1;
+ private activeSplit: 'left' | 'right' | null = null;
+
+ constructor(asset: Asset) {
+ this.asset = asset;
+ this.init();
+ }
+
+ private async init() {
+ if (this.ready) return;
+ const transcript = await resolveTranscript(this.asset);
+ this.groups = groupBy(transcript, { length: 18 });
+ this.ready = true;
+ }
+
+ public reposition(world: World, entity: Entity): boolean {
+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.CENTER });
+ }
+
+ public applyStyles(world: World, entity: Entity): void {
+ if (!this.reposition(world, entity)) return;
+
+ entity.add(TextStyle);
+ entity.set(TextStyle, {
+ fontFamily: 'Montserrat',
+ fontWeight: '300',
+ fontSize: 50,
+ textAlign: TextAlign.CENTER,
+ textBaseline: TextBaseline.MIDDLE,
+ textCase: TextCase.ORIGINAL,
+ leading: 0.9,
+ fontStyle: FontStyle.NORMAL,
+ });
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: 0xFFFFFF });
+ appendChild(world, fill, entity);
+
+ loadWebFont(world, 'Montserrat');
+ }
+
+ public seekTo(world: World, entity: Entity, relativeTime: number): void {
+ const groupIndex = findActiveGroup(this.groups, relativeTime);
+
+ if (groupIndex === -1) {
+ setChars(world, entity, '');
+ clearTextRanges(world, entity);
+ this.currentGroupIndex = -1;
+ this.activeSplit = null;
+ return;
+ }
+
+ const group = this.groups[groupIndex]!;
+ const [left, right] = splitSequence(group);
+ const leftText = left.map(w => w.text).join(' ');
+ const rightText = right.map(w => w.text).join(' ');
+ const text = rightText ? `${leftText}\n${rightText}` : leftText;
+ const activeSplit = right.length > 0 && relativeTime >= right[0]!.start ? 'right' : 'left';
+
+ if (groupIndex !== this.currentGroupIndex || activeSplit !== this.activeSplit) {
+ this.currentGroupIndex = groupIndex;
+ this.activeSplit = activeSplit;
+ setChars(world, entity, text);
+
+ // Two-line display with newline separator.
+ // The active line gets weight '500', inactive stays at the node's '300'.
+ clearTextRanges(world, entity);
+ const start = activeSplit === 'left' ? 0 : leftText.length + 1;
+ const end = activeSplit === 'left' ? leftText.length : text.length;
+ if (end > start) {
+ const range = createEntity(world);
+ range.add(TextRange);
+ range.set(TextRange, { start, end });
+ range.add(TextStyle);
+ range.set(TextStyle, { fontWeight: '500' });
+ appendChild(world, range, entity);
+ }
+ }
+ }
+
+ public draw(world: World, entity: Entity): void {
+ renderText(world, entity);
+ }
+
+ public dispose(): void {
+ this.groups = [];
+ this.currentGroupIndex = -1;
+ this.activeSplit = null;
+ }
+}
diff --git a/packages/runtime/src/media/caption/position.ts b/packages/runtime/src/media/caption/position.ts
new file mode 100644
index 00000000..9d9be1c6
--- /dev/null
+++ b/packages/runtime/src/media/caption/position.ts
@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { store } from '../../world/store';
+import { CaptionAlign } from '../../constants';
+import { Caption, Position, Computed } from '../../traits';
+import { getParentNode } from '../../queries/hierarchy';
+import { resizeEntity } from '../../actions/resize';
+
+import type { Entity, World } from 'koota';
+
+// Margin between the caption box and the parent edge for top/bottom alignment.
+export const CAPTION_MARGIN = 100;
+
+/**
+ * Sizes the caption box and places it within its parent. Horizontal placement
+ * belongs to the preset (centered unless a fixed `x` is given); vertical
+ * placement follows the entity's `Caption.verticalAlign`, falling back to the
+ * preset's default. Returns false when the entity has no parent to place
+ * against.
+ */
+export function placeCaption(
+ world: World,
+ entity: Entity,
+ preset: { width: number; height: number; x?: number; defaultAlign: CaptionAlign },
+): boolean {
+ const parent = getParentNode(entity);
+ if (parent === null) return false;
+ const computed = store(world, Computed);
+ const parentWidth = computed.width[parent.id()]!;
+ const parentHeight = computed.height[parent.id()]!;
+
+ resizeEntity(world, entity, { width: preset.width, height: preset.height });
+
+ const align = entity.get(Caption)?.verticalAlign ?? preset.defaultAlign;
+ const y = align === CaptionAlign.TOP
+ ? CAPTION_MARGIN
+ : align === CaptionAlign.BOTTOM
+ ? parentHeight - preset.height - CAPTION_MARGIN
+ : (parentHeight - preset.height) / 2;
+
+ entity.add(Position);
+ entity.set(Position, {
+ x: preset.x ?? (parentWidth - preset.width) / 2,
+ y,
+ });
+ return true;
+}
diff --git a/packages/runtime/src/media/caption/spotlight.ts b/packages/runtime/src/media/caption/spotlight.ts
new file mode 100644
index 00000000..c90ffd25
--- /dev/null
+++ b/packages/runtime/src/media/caption/spotlight.ts
@@ -0,0 +1,137 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { store } from '../../world/store';
+import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';
+import { Paint, Color, Caption, TextRange, TextStyle } from '../../traits';
+import { renderText } from '../../utils/text';
+import { loadWebFont } from '../../fonts/utils';
+import { groupBy, findActiveGroup, clearTextRanges, resolveTranscript, setChars } from './utils';
+import { placeCaption } from './position';
+import { createEntity } from '../../actions/entities';
+import { appendChild } from '../../actions/hierarchy';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '../../assets/types';
+import type { CaptionDecoder } from './types';
+
+
+const WIDTH = 700;
+const HEIGHT = 100;
+const HIGHLIGHT_COLOR = 0x24D5FF;
+
+export class SpotlightCaptionDecoder implements CaptionDecoder {
+ public readonly type = CaptionType.SPOTLIGHT;
+ public groups: ReturnType<typeof groupBy> = [];
+ public ready = false;
+
+ private readonly asset: Asset;
+ private currentGroupIndex = -1;
+ private currentWordIndex = -1;
+ private fill: Entity | null = null;
+
+ constructor(asset: Asset) {
+ this.asset = asset;
+ this.init();
+ }
+
+ private async init() {
+ if (this.ready) return;
+ const transcript = await resolveTranscript(this.asset);
+ this.groups = groupBy(transcript, { length: 10 });
+ this.ready = true;
+ }
+
+ public reposition(world: World, entity: Entity): boolean {
+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.CENTER });
+ }
+
+ public applyStyles(world: World, entity: Entity): void {
+ if (!this.reposition(world, entity)) return;
+
+ entity.add(TextStyle);
+ entity.set(TextStyle, {
+ fontFamily: 'The Bold Font',
+ fontWeight: '500',
+ fontStyle: FontStyle.ITALIC,
+ fontSize: 70,
+ textAlign: TextAlign.CENTER,
+ textBaseline: TextBaseline.MIDDLE,
+ textCase: TextCase.ORIGINAL,
+ leading: 1,
+ });
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: 0xFFFFFF });
+ appendChild(world, fill, entity);
+
+ loadWebFont(world, 'The Bold Font');
+ }
+
+ public seekTo(world: World, entity: Entity, relativeTime: number): void {
+ const groupIndex = findActiveGroup(this.groups, relativeTime);
+
+ if (groupIndex === -1) {
+ setChars(world, entity, '');
+ clearTextRanges(world, entity);
+ this.fill = null;
+ this.currentGroupIndex = -1;
+ this.currentWordIndex = -1;
+ return;
+ }
+
+ const group = this.groups[groupIndex]!;
+
+ const wordIndex = group.findIndex(word =>
+ relativeTime >= word.start && relativeTime <= word.end
+ );
+
+ const text = group.map(w => w.text).join(' ');
+
+ if (groupIndex !== this.currentGroupIndex || wordIndex !== this.currentWordIndex) {
+ this.currentGroupIndex = groupIndex;
+ this.currentWordIndex = wordIndex;
+ setChars(world, entity, text);
+
+ clearTextRanges(world, entity);
+ this.fill = null;
+
+ if (group.length > 1 && wordIndex !== -1) {
+ const start = group.slice(0, wordIndex).map(w => w.text).join(' ').length;
+ const end = group.slice(0, wordIndex + 1).map(w => w.text).join(' ').length;
+ const range = createEntity(world);
+ range.add(TextRange);
+ range.set(TextRange, { start, end });
+ appendChild(world, range, entity);
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: HIGHLIGHT_COLOR });
+ appendChild(world, fill, range);
+ this.fill = fill;
+ }
+ }
+ }
+
+ public draw(world: World, entity: Entity): void {
+ const highlightColor = entity.get(Caption)?.colors?.[0] ?? HIGHLIGHT_COLOR;
+
+ if (this.fill) {
+ store(world, Color).value[this.fill.id()] = highlightColor;
+ }
+
+ renderText(world, entity);
+ }
+
+ public dispose(): void {
+ this.groups = [];
+ this.currentGroupIndex = -1;
+ this.currentWordIndex = -1;
+ }
+}
diff --git a/packages/runtime/src/media/caption/stark.ts b/packages/runtime/src/media/caption/stark.ts
new file mode 100644
index 00000000..4714973e
--- /dev/null
+++ b/packages/runtime/src/media/caption/stark.ts
@@ -0,0 +1,95 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { BlendMode, CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';
+import { Paint, Color, Appearance, TextStyle } from '../../traits';
+import { renderText } from '../../utils/text';
+import { loadWebFont } from '../../fonts/utils';
+import { groupBy, findActiveGroup, resolveTranscript, setChars } from './utils';
+import { placeCaption } from './position';
+import { createEntity } from '../../actions/entities';
+import { appendChild } from '../../actions/hierarchy';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '../../assets/types';
+import type { CaptionDecoder } from './types';
+
+const WIDTH = 700;
+const HEIGHT = 100;
+
+export class StarkCaptionDecoder implements CaptionDecoder {
+ public readonly type = CaptionType.STARK;
+ public groups: ReturnType<typeof groupBy> = [];
+ public ready = false;
+
+ private readonly asset: Asset;
+ private currentGroupIndex = -1;
+
+ constructor(asset: Asset) {
+ this.asset = asset;
+ this.init();
+ }
+
+ private async init() {
+ if (this.ready) return;
+ const transcript = await resolveTranscript(this.asset);
+ this.groups = groupBy(transcript, { duration: 0.2 });
+ this.ready = true;
+ }
+
+ public reposition(world: World, entity: Entity): boolean {
+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.CENTER });
+ }
+
+ public applyStyles(world: World, entity: Entity): void {
+ if (!this.reposition(world, entity)) return;
+
+ entity.add(TextStyle);
+ entity.set(TextStyle, {
+ fontFamily: 'Figtree',
+ fontWeight: '800',
+ fontSize: 70,
+ textAlign: TextAlign.CENTER,
+ textBaseline: TextBaseline.MIDDLE,
+ textCase: TextCase.UPPER,
+ fontStyle: FontStyle.NORMAL,
+ leading: 1,
+ });
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: 0xFFFFFF });
+ fill.add(Appearance);
+ fill.set(Appearance, { blendMode: BlendMode.DIFFERENCE });
+ appendChild(world, fill, entity);
+
+ loadWebFont(world, 'Figtree');
+ }
+
+ public seekTo(world: World, entity: Entity, relativeTime: number): void {
+ const groupIndex = findActiveGroup(this.groups, relativeTime);
+
+ if (groupIndex === -1) {
+ setChars(world, entity, '');
+ this.currentGroupIndex = -1;
+ return;
+ }
+
+ if (groupIndex !== this.currentGroupIndex) {
+ this.currentGroupIndex = groupIndex;
+ setChars(world, entity, this.groups[groupIndex]!.map(w => w.text).join(' '));
+ }
+ }
+
+ public draw(world: World, entity: Entity): void {
+ renderText(world, entity);
+ }
+
+ public dispose(): void {
+ this.groups = [];
+ this.currentGroupIndex = -1;
+ }
+}
diff --git a/packages/runtime/src/media/caption/subtitles.ts b/packages/runtime/src/media/caption/subtitles.ts
new file mode 100644
index 00000000..8f9ab2a2
--- /dev/null
+++ b/packages/runtime/src/media/caption/subtitles.ts
@@ -0,0 +1,60 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { Transcript } from '../../assets/types';
+
+const TIMESTAMP_RE = /(?:\d{1,2}:)?\d{1,2}:\d{2}[.,]\d{1,3}/g;
+
+function parseTimestamp(raw: string): number {
+ return raw
+ .replace(',', '.')
+ .split(':')
+ .reduce((acc, part) => acc * 60 + Number(part), 0);
+}
+
+/**
+ * Parses SRT or WebVTT text into a `Transcript`. Each cue becomes one segment
+ * (a sentence boundary for grouping). Cues carry no word timings, so each
+ * word's window is synthesized proportional to its length within the cue.
+ */
+export function parseSubtitles(text: string): Transcript {
+ const transcript: Transcript = [];
+ const blocks = text.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n').split(/\n{2,}/);
+
+ for (const block of blocks) {
+ const lines = block.split('\n');
+ const timingIndex = lines.findIndex((line) => line.includes('-->'));
+ if (timingIndex === -1) continue; // WEBVTT header, NOTE/STYLE/REGION blocks, stray indices
+
+ const timestamps = lines[timingIndex].match(TIMESTAMP_RE);
+ if (!timestamps || timestamps.length < 2) continue;
+ const start = parseTimestamp(timestamps[0]);
+ const end = parseTimestamp(timestamps[1]);
+ if (!(end > start)) continue;
+
+ // Strip inline markup: HTML/VTT tags (<i>, <c.class>, <00:01.000>) and
+ // SSA-style overrides ({\an8}).
+ const content = lines
+ .slice(timingIndex + 1)
+ .join(' ')
+ .replace(/<[^>]*>/g, '')
+ .replace(/\{[^}]*\}/g, '');
+
+ const tokens = content.split(/\s+/).filter(Boolean);
+ if (tokens.length === 0) continue;
+
+ const total = tokens.reduce((acc, token) => acc + token.length, 0);
+ const duration = end - start;
+ let elapsed = 0;
+ const words = tokens.map((token) => {
+ const wordStart = start + (elapsed / total) * duration;
+ elapsed += token.length;
+ return { text: token, start: wordStart, end: start + (elapsed / total) * duration };
+ });
+
+ transcript.push({ text: tokens.join(' '), words });
+ }
+
+ return transcript.sort((a, b) => a.words[0].start - b.words[0].start);
+}
diff --git a/packages/runtime/src/media/caption/types.ts b/packages/runtime/src/media/caption/types.ts
new file mode 100644
index 00000000..c4f3a0e6
--- /dev/null
+++ b/packages/runtime/src/media/caption/types.ts
@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { Entity, World } from 'koota';
+import type { CaptionType } from '../../constants';
+import type { WordGroup } from '../../assets/types';
+
+export interface CaptionDecoder {
+ readonly type: CaptionType;
+ groups: WordGroup[];
+ ready: boolean;
+ applyStyles(world: World, entity: Entity): void;
+ reposition(world: World, entity: Entity): boolean;
+ seekTo(world: World, entity: Entity, relativeTime: number): void;
+ draw(world: World, entity: Entity): void;
+ dispose(): void;
+}
diff --git a/packages/runtime/src/media/caption/utils.ts b/packages/runtime/src/media/caption/utils.ts
new file mode 100644
index 00000000..57627a0d
--- /dev/null
+++ b/packages/runtime/src/media/caption/utils.ts
@@ -0,0 +1,133 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { store } from '../../world/store';
+import { ChildOf, TextRange, Cache, Chars } from '../../traits';
+import { deleteEntity } from '../../actions/entities';
+import { getAssetFile } from '../../actions/assets';
+import { parseSubtitles } from './subtitles';
+
+import type { Entity, World } from 'koota';
+import type { Asset, Transcript, WordGroup } from '../../assets/types';
+
+export type TextRangeOverride = {
+ start: number;
+ end: number;
+};
+
+
+export type GroupByOptions = {
+ duration: number;
+} | {
+ length: number;
+};
+
+const ENDING_PUNCTUATION = ['.', '!', '?', ',', ';'];
+
+export function groupBy(transcript: Transcript, options: GroupByOptions): WordGroup[] {
+ const groups: WordGroup[] = [[]];
+
+ for (const segment of transcript) {
+ // Segment boundaries are sentence boundaries — never merge across them.
+ if (groups[groups.length - 1]!.length > 0) {
+ groups.push([]);
+ }
+
+ for (const word of segment.words) {
+ const last = groups[groups.length - 1]!;
+ const charCount = last.reduce((acc, w) => acc + w.text.length, 0);
+ const duration = last.reduce((acc, w) => acc + (w.end - w.start), 0);
+
+ if ('duration' in options && duration + (word.end - word.start) > options.duration) {
+ groups.push([]);
+ } else if ('length' in options && charCount + word.text.length > options.length) {
+ groups.push([]);
+ }
+
+ groups.at(-1)?.push(word);
+
+ if (ENDING_PUNCTUATION.some(p => word.text.endsWith(p))) {
+ groups.push([]);
+ }
+ }
+ }
+
+ return groups.filter(g => g.length > 0);
+}
+
+/**
+ * Find the active group index for a given time.
+ * Returns -1 if no group is active.
+ */
+export function findActiveGroup(groups: WordGroup[], relativeTime: number): number {
+ return groups.findIndex(group =>
+ relativeTime >= group[0]!.start && relativeTime <= (group.at(-1)?.end ?? 0)
+ );
+}
+
+/**
+ * Playhead-driven caption text: a raw store write, no change events, so the
+ * app never persists per-frame caption content (bitecs wrote the array too).
+ */
+export function setChars(world: World, entity: Entity, text: string) {
+ if (!entity.has(Chars)) entity.add(Chars);
+ store(world, Chars).value[entity.id()] = text;
+}
+
+export function clearTextRanges(world: World, parent: Entity) {
+ for (const range of world.query(ChildOf(parent), TextRange)) {
+ deleteEntity(world, range);
+ }
+
+ if (parent.has(Cache)) {
+ store(world, Cache).textRanges[parent.id()] = [];
+ }
+}
+
+/**
+ * Split a word group into two halves at the nearest word boundary
+ * to the midpoint of the joined text. Used by paper and guinea presets.
+ */
+export function splitSequence(sequence: WordGroup): [WordGroup, WordGroup] {
+ const text = sequence.map(w => w.text).join(' ');
+ const midPoint = Math.ceil(text.length / 2);
+
+ let index = text.length;
+ for (let i = midPoint, j = midPoint; i > 0 && j < text.length - 1; i--, j++) {
+ if (text[i] === ' ') {
+ index = i;
+ break;
+ }
+ if (text[j] === ' ') {
+ index = j;
+ break;
+ }
+ }
+
+ const leftText = text.slice(0, index).trim();
+ const splitAt = leftText.split(/ /).length;
+
+ return [sequence.slice(0, splitAt), sequence.slice(splitAt)];
+}
+
+
+/**
+ * Resolve a transcript from any asset type that can carry one.
+ */
+export async function resolveTranscript(asset: Asset): Promise<Transcript> {
+ if (asset.type === 'TRANSCRIPT') {
+ const file = await getAssetFile(asset);
+ const text = await file.text();
+ if (asset.mimeType === 'application/x-subrip' || asset.mimeType === 'text/vtt') {
+ return parseSubtitles(text);
+ }
+ return JSON.parse(text);
+ }
+
+ if ((asset.type === 'AUDIO' || asset.type === 'VIDEO') && asset.transcript) {
+ return asset.transcript;
+ }
+
+ return [];
+}
diff --git a/packages/runtime/src/media/caption/whisper.ts b/packages/runtime/src/media/caption/whisper.ts
new file mode 100644
index 00000000..1ed00959
--- /dev/null
+++ b/packages/runtime/src/media/caption/whisper.ts
@@ -0,0 +1,94 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';
+import { Paint, Color, TextStyle } from '../../traits';
+import { renderText } from '../../utils/text';
+import { loadWebFont } from '../../fonts/utils';
+import { groupBy, findActiveGroup, resolveTranscript, setChars } from './utils';
+import { placeCaption } from './position';
+import { createEntity } from '../../actions/entities';
+import { appendChild } from '../../actions/hierarchy';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '../../assets/types';
+import type { CaptionDecoder } from './types';
+
+const WIDTH = 1000;
+const HEIGHT = 100;
+
+export class WhisperCaptionDecoder implements CaptionDecoder {
+ public readonly type = CaptionType.WHISPER;
+ public groups: ReturnType<typeof groupBy> = [];
+ public ready = false;
+
+ private readonly asset: Asset;
+ private currentGroupIndex = -1;
+
+ constructor(asset: Asset) {
+ this.asset = asset;
+ this.init();
+ }
+
+ private async init() {
+ if (this.ready) return;
+ const transcript = await resolveTranscript(this.asset);
+ this.groups = groupBy(transcript, { duration: 2 });
+
+ this.ready = true;
+ }
+
+ public reposition(world: World, entity: Entity): boolean {
+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.BOTTOM });
+ }
+
+ public applyStyles(world: World, entity: Entity): void {
+ if (!this.reposition(world, entity)) return;
+
+ entity.add(TextStyle);
+ entity.set(TextStyle, {
+ fontFamily: 'Montserrat',
+ fontWeight: '400',
+ fontSize: 40,
+ textAlign: TextAlign.CENTER,
+ textBaseline: TextBaseline.MIDDLE,
+ textCase: TextCase.ORIGINAL,
+ leading: 1.4,
+ fontStyle: FontStyle.NORMAL,
+ });
+
+ const fill = createEntity(world);
+ fill.add(Paint);
+ fill.set(Paint, { value: PaintType.SOLID });
+ fill.add(Color);
+ fill.set(Color, { value: 0xFFFFFF });
+ appendChild(world, fill, entity);
+
+ loadWebFont(world, 'Montserrat');
+ }
+
+ public seekTo(world: World, entity: Entity, relativeTime: number): void {
+ const groupIndex = findActiveGroup(this.groups, relativeTime);
+
+ if (groupIndex === -1) {
+ setChars(world, entity, '');
+ this.currentGroupIndex = -1;
+ return;
+ }
+
+ if (groupIndex !== this.currentGroupIndex) {
+ this.currentGroupIndex = groupIndex;
+ setChars(world, entity, this.groups[groupIndex]!.map(w => w.text).join(' '));
+ }
+ }
+
+ public draw(world: World, entity: Entity): void {
+ renderText(world, entity);
+ }
+
+ public dispose(): void {
+ this.groups = [];
+ this.currentGroupIndex = -1;
+ }
+}
diff --git a/packages/runtime/src/media/dispose.ts b/packages/runtime/src/media/dispose.ts
new file mode 100644
index 00000000..1e890e2b
--- /dev/null
+++ b/packages/runtime/src/media/dispose.ts
@@ -0,0 +1,87 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Subtree walkers releasing live runtime handles (was part of api/utils.ts).
+// They deliberately include Deleted descendants: a tombstoned subtree keeps
+// its records for undo, but its decoders and buses must not linger.
+
+import {
+ ChildOf,
+ ImageDecoderHandle, VideoDecoderHandle, SequenceDecoderHandle,
+ AudioDecoderHandle, CaptionDecoderHandle,
+ HtmlHostHandle, SurfaceHostHandle, ShaderHostHandle, AudioBusHandle,
+} from '../traits';
+
+import type { Entity, World } from 'koota';
+
+export function disposeDecoders(world: World, entity: Entity): void {
+ if (entity.has(ImageDecoderHandle)) {
+ entity.get(ImageDecoderHandle)?.dispose();
+ entity.set(ImageDecoderHandle, null);
+ }
+ if (entity.has(VideoDecoderHandle)) {
+ entity.get(VideoDecoderHandle)?.dispose();
+ entity.set(VideoDecoderHandle, null);
+ }
+ if (entity.has(SequenceDecoderHandle)) {
+ entity.get(SequenceDecoderHandle)?.dispose();
+ entity.set(SequenceDecoderHandle, null);
+ }
+ if (entity.has(AudioDecoderHandle)) {
+ entity.get(AudioDecoderHandle)?.reset();
+ entity.set(AudioDecoderHandle, null);
+ }
+ if (entity.has(CaptionDecoderHandle)) {
+ entity.get(CaptionDecoderHandle)?.dispose();
+ entity.set(CaptionDecoderHandle, null);
+ }
+
+ for (const child of world.query(ChildOf(entity))) {
+ disposeDecoders(world, child);
+ }
+}
+
+export function disposeHtmlHosts(world: World, entity: Entity): void {
+ if (entity.has(HtmlHostHandle)) {
+ entity.get(HtmlHostHandle)?.dispose();
+ entity.set(HtmlHostHandle, null);
+ }
+
+ for (const child of world.query(ChildOf(entity))) {
+ disposeHtmlHosts(world, child);
+ }
+}
+
+export function disposeSurfaceHosts(world: World, entity: Entity): void {
+ if (entity.has(SurfaceHostHandle)) {
+ entity.get(SurfaceHostHandle)?.dispose();
+ entity.set(SurfaceHostHandle, null);
+ }
+
+ for (const child of world.query(ChildOf(entity))) {
+ disposeSurfaceHosts(world, child);
+ }
+}
+
+export function disposeShaderHosts(world: World, entity: Entity): void {
+ if (entity.has(ShaderHostHandle)) {
+ entity.get(ShaderHostHandle)?.dispose();
+ entity.set(ShaderHostHandle, null);
+ }
+
+ for (const child of world.query(ChildOf(entity))) {
+ disposeShaderHosts(world, child);
+ }
+}
+
+export function disconnectAudioBus(world: World, entity: Entity): void {
+ if (entity.has(AudioBusHandle)) {
+ entity.get(AudioBusHandle)?.disconnect();
+ entity.set(AudioBusHandle, null);
+ }
+
+ for (const child of world.query(ChildOf(entity))) {
+ disconnectAudioBus(world, child);
+ }
+}
diff --git a/packages/runtime/src/media/frame-cache.ts b/packages/runtime/src/media/frame-cache.ts
new file mode 100644
index 00000000..7630b459
--- /dev/null
+++ b/packages/runtime/src/media/frame-cache.ts
@@ -0,0 +1,186 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+type FrameCacheConfig = {
+ pixels: number;
+ count: number;
+}
+
+type FrameTile = {
+ tileIndex: number;
+ frameIndex: number;
+}
+
+type Frame = VideoFrame | ImageBitmap;
+
+function getDisplaySize(frame: Frame) {
+ if (frame instanceof VideoFrame) {
+ return [frame.displayWidth, frame.displayHeight];
+ }
+ return [frame.width, frame.height]
+}
+
+export class FrameCache {
+ public readonly config: FrameCacheConfig;
+ public readonly atlas = new OffscreenCanvas(0, 0);
+ public readonly atlasCtx = this.atlas.getContext('2d')!;
+ public readonly source = new OffscreenCanvas(0, 0);
+ public readonly sourceCtx = this.source.getContext('2d')!;
+
+ private tileWidth: number = 0;
+ private tileHeight: number = 0;
+ private columns: number = 0;
+ private tiles: FrameTile[];
+
+ public rotation: number = 0;
+ public leftFrameIndex: number;
+ public rightFrameIndex: number;
+ public lastInserted: number = -1;
+
+ public constructor(config: FrameCacheConfig) {
+ this.config = config;
+ this.leftFrameIndex = 0;
+ this.rightFrameIndex = config.count - 1;
+ this.tiles = [];
+ }
+
+ private resizeCaches(frame: Frame) {
+ const rotation = this.rotation;
+ const [width, height] = getDisplaySize(frame);
+ const pixelCount = width * height;
+
+ const factor = Math.min(1, Math.sqrt(this.config.pixels / pixelCount));
+
+ const sourceWidth = rotation === 90 || rotation === 270
+ ? height
+ : width;
+
+ const sourceHeight = rotation === 90 || rotation === 270
+ ? width
+ : height;
+
+ this.tileWidth = Math.floor(sourceWidth * factor);
+ this.tileHeight = Math.floor(sourceHeight * factor);
+
+ // Lay tiles in a 2D grid
+ this.columns = Math.ceil(Math.sqrt(this.config.count));
+ const rows = Math.ceil(this.config.count / this.columns);
+
+ const absoluteWidth = this.tileWidth * this.columns;
+ const absoluteHeight = this.tileHeight * rows;
+
+ if (this.atlas.width !== absoluteWidth || this.atlas.height !== absoluteHeight) {
+ this.atlas.width = absoluteWidth;
+ this.atlas.height = absoluteHeight;
+ this.atlasCtx.imageSmoothingQuality = 'high';
+ this.tiles = []; // setting the canvas size resets the tiles
+ }
+ }
+
+ private drawRotated(frame: Frame) {
+ const ctx = this.sourceCtx;
+ const rotation = this.rotation;
+
+ const [displayWidth, displayHeight] = getDisplaySize(frame);
+
+ const width = rotation === 90 || rotation === 270
+ ? displayHeight
+ : displayWidth;
+
+ const height = rotation === 90 || rotation === 270
+ ? displayWidth
+ : displayHeight;
+
+ if (this.source.width !== width || this.source.height !== height) {
+ this.source.width = width;
+ this.source.height = height;
+ ctx.imageSmoothingEnabled = false;
+ }
+
+ ctx.resetTransform();
+ ctx.clearRect(0, 0, width, height);
+ ctx.translate(width / 2, height / 2);
+ ctx.rotate((rotation * Math.PI) / 180);
+ ctx.drawImage(frame, -displayWidth / 2, -displayHeight / 2, displayWidth, displayHeight);
+ }
+
+ private evictTiles() {
+ if (this.tiles.length < this.config.count) return;
+
+ this.tiles = this.tiles.filter(t => t.frameIndex >= this.leftFrameIndex && t.frameIndex <= this.rightFrameIndex);
+ }
+
+ private insertTile(frameIndex: number) {
+ let tileIndex = 0;
+ while (this.tiles.some(t => t.tileIndex === tileIndex)) {
+ tileIndex++;
+ }
+
+ this.tiles.push({ tileIndex, frameIndex });
+
+ const ctx = this.atlasCtx;
+ const x = (tileIndex % this.columns) * this.tileWidth;
+ const y = Math.floor(tileIndex / this.columns) * this.tileHeight;
+
+ ctx.clearRect(x, y, this.tileWidth, this.tileHeight);
+ ctx.drawImage(this.source, x, y, this.tileWidth, this.tileHeight);
+ }
+
+ public has(frameIndex: number) {
+ return this.tiles.some(t => t.frameIndex === frameIndex);
+ }
+
+ /**
+ * Frame index of the cached frame closest to `frameIndex`, or `undefined` when
+ * nothing is cached within `tolerance`. Ties resolve to the earlier frame, so a
+ * playhead sitting between two cached frames never shows one that hasn't played yet.
+ */
+ public findNearest(frameIndex: number, tolerance: number): number | undefined {
+ let best: number | undefined;
+ let bestDistance = tolerance + 1;
+
+ for (const tile of this.tiles) {
+ const distance = Math.abs(tile.frameIndex - frameIndex);
+ if (distance < bestDistance || (best !== undefined && distance === bestDistance && tile.frameIndex < best)) {
+ bestDistance = distance;
+ best = tile.frameIndex;
+ }
+ }
+
+ return best;
+ }
+
+ public insert(frame: VideoFrame | ImageBitmap, index: number) {
+ if (this.has(index)) return;
+
+ this.evictTiles();
+ this.resizeCaches(frame);
+ this.drawRotated(frame);
+ this.insertTile(index);
+ this.lastInserted = index;
+ }
+
+ public findTile(frameIndex: number) {
+ const tile = this.tiles.find(t => t.frameIndex === frameIndex);
+
+ if (!tile) return;
+
+ const x = (tile.tileIndex % this.columns) * this.tileWidth;
+ const y = Math.floor(tile.tileIndex / this.columns) * this.tileHeight;
+
+ return { x, y, width: this.tileWidth, height: this.tileHeight };
+ }
+
+ public dispose() {
+ this.atlas.width = 0;
+ this.atlas.height = 0;
+ this.source.width = 0;
+ this.source.height = 0;
+ this.tileWidth = 0;
+ this.tileHeight = 0;
+ this.columns = 0;
+ this.lastInserted = -1;
+ this.tiles = [];
+ }
+}
diff --git a/packages/runtime/src/media/html.ts b/packages/runtime/src/media/html.ts
new file mode 100644
index 00000000..ec4de380
--- /dev/null
+++ b/packages/runtime/src/media/html.ts
@@ -0,0 +1,152 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+
+// html-in-canvas (https://github.com/WICG/html-in-canvas). Chromium only,
+// behind chrome://flags/#canvas-draw-element; the API surface is still
+// moving, so every touchpoint is typed and isolated here.
+type DrawElementContext = Ctx2D & {
+ drawElementImage(source: Element | unknown, dx: number, dy: number, dw: number, dh: number): DOMMatrix;
+};
+
+export function isHtmlInCanvasSupported(): boolean {
+ return typeof CanvasRenderingContext2D !== 'undefined'
+ && 'drawElementImage' in CanvasRenderingContext2D.prototype;
+}
+
+/**
+ * Resolves after the browser's next rendering update (style/layout/paint) has completed.
+ */
+export function nextRenderingUpdate(): Promise<void> {
+ return new Promise(resolve => {
+ // timeout in case requestAnimationFrame is not called
+ const timeout = setTimeout(resolve, 250);
+
+ requestAnimationFrame(() => {
+ clearTimeout(timeout);
+ setTimeout(resolve, 0);
+ });
+ });
+}
+
+let htmlCanvas: HTMLCanvasElement | null = null;
+let htmlCtx: CanvasRenderingContext2D | null = null;
+let htmlHosts = 0;
+
+function acquireLayoutCanvas(): HTMLCanvasElement {
+ if (!htmlCanvas) {
+ htmlCanvas = document.createElement('canvas');
+ htmlCanvas.width = 1;
+ htmlCanvas.height = 1;
+ htmlCanvas.toggleAttribute('layoutsubtree', true);
+ htmlCanvas.style.cssText = 'position:fixed;left:0;top:0;width:1px;height:1px;pointer-events:none;';
+ document.body.appendChild(htmlCanvas);
+ htmlCtx = htmlCanvas.getContext('2d')!;
+ }
+ htmlHosts++;
+ return htmlCanvas;
+}
+
+function releaseLayoutCanvas(): void {
+ if (--htmlHosts === 0) {
+ htmlCanvas?.remove();
+ htmlCanvas = null;
+ htmlCtx = null;
+ }
+}
+
+/**
+ * DOM host for one HTML paint: a div inside the shared layoutsubtree canvas.
+ * The mount document inserts the paint's JSX children into `element` as real,
+ * reactive DOM; the browser lays them out at the parent geometry's box size
+ * and `draw` rasterizes the result at the destination's device scale and
+ * blits it into the rendering context (stage or offscreen alike).
+ */
+export class HtmlHost {
+ public readonly element: HTMLDivElement;
+ private disposed = false;
+ private warned = false;
+ private width = -1;
+ private height = -1;
+
+ public constructor() {
+ const el = document.createElement('div');
+ // pointer-events: none — layoutsubtree children participate in hit
+ // testing and would otherwise swallow page interactions.
+ el.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';
+ acquireLayoutCanvas().appendChild(el);
+ this.element = el;
+ }
+
+ /**
+ * Resolves asynchronously loaded html resources.
+ */
+ public whenReady(timeInSeconds: number): Promise<void> {
+ // sync animations
+ for (const animation of this.element.getAnimations({ subtree: true })) {
+ if (animation.playState !== 'paused') animation.pause();
+ animation.currentTime = timeInSeconds * 1000;
+ }
+
+ // wait for fonts to be ready
+ const pending: Promise<unknown>[] = [document.fonts.ready];
+
+ // wait for images to be ready
+ for (const image of this.element.querySelectorAll('img')) {
+ // `complete` covers loaded, failed, and srcless images alike; the
+ // rest are still in flight, and a failed decode just paints nothing.
+ if (image.complete) continue;
+ pending.push(image.decode().catch(() => undefined));
+ }
+
+ return Promise.all(pending).then(() => nextRenderingUpdate());
+ }
+
+ public setSize(width: number, height: number): void {
+ if (width === this.width && height === this.height) return;
+ this.width = width;
+ this.height = height;
+ this.element.style.width = `${width}px`;
+ this.element.style.height = `${height}px`;
+ }
+
+ public draw(ctx: Ctx2D, width: number, height: number): void {
+ if (this.disposed || !htmlCanvas || !htmlCtx) return;
+
+ this.setSize(width, height);
+
+ try {
+ const m = ctx.getTransform();
+ const sx = Math.hypot(m.a, m.b) || 1;
+ const sy = Math.hypot(m.c, m.d) || 1;
+ const pw = Math.max(1, Math.ceil(width * sx));
+ const ph = Math.max(1, Math.ceil(height * sy));
+
+ // Grow-only: shrinking would clear and reallocate every time hosts
+ // of different sizes share the canvas within one frame.
+ if (htmlCanvas.width < pw) htmlCanvas.width = pw;
+ if (htmlCanvas.height < ph) htmlCanvas.height = ph;
+
+ htmlCtx.setTransform(sx, 0, 0, sy, 0, 0);
+ htmlCtx.clearRect(0, 0, width, height);
+ (htmlCtx as DrawElementContext).drawElementImage(this.element, 0, 0, width, height);
+ ctx.drawImage(htmlCanvas, 0, 0, pw, ph, 0, 0, width, height);
+ } catch (e) {
+ // A transient failure (subtree not yet laid out or painted) just
+ // skips the frame; log once so a permanent one doesn't spam.
+ if (!this.warned) {
+ this.warned = true;
+ console.error(`Error drawing <HtmlPaint> content: ${e instanceof Error ? `${e.name}: ${e.message}` : e}`);
+ }
+ }
+ }
+
+ public dispose(): void {
+ if (this.disposed) return;
+ this.disposed = true;
+ this.element.remove();
+ releaseLayoutCanvas();
+ }
+}
diff --git a/packages/runtime/src/media/image.ts b/packages/runtime/src/media/image.ts
new file mode 100644
index 00000000..4bd45a3f
--- /dev/null
+++ b/packages/runtime/src/media/image.ts
@@ -0,0 +1,258 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { AssetId, ImageDecoderHandle, Assets } from '../traits';
+import { getAssetFile } from '../actions/assets';
+
+import type { Entity, World } from 'koota';
+import type { ImageAsset } from '../assets/types';
+
+export type DecodedImage = ImageBitmap | HTMLImageElement;
+
+export interface ImageDecoder {
+ readonly assetId: string;
+ readonly failed: boolean;
+ readonly ready: boolean;
+ init(): Promise<void>;
+ getBitmap(targetWidth: number, targetHeight: number): DecodedImage | null;
+ dispose(): void;
+}
+
+/**
+ * Image decoder backed by `createImageBitmap`. Used for raster formats
+ * (PNG, JPEG, WebP, …) where we want a GPU-friendly bitmap.
+ *
+ * Fetching + decoding starts immediately on construction.
+ */
+export class BitmapImageDecoder implements ImageDecoder {
+ readonly assetId: string;
+ public failed = false;
+ private bitmap: ImageBitmap | null = null;
+ private disposed = false;
+
+ public constructor(private readonly asset: ImageAsset) {
+ this.assetId = asset.id;
+ }
+
+ public async init() {
+ if (this.disposed) return;
+
+ try {
+ const file = await getAssetFile(this.asset);
+ const result = await createImageBitmap(file);
+ if (this.disposed) {
+ result.close();
+ return;
+ }
+ this.bitmap = result;
+ } catch (e) {
+ console.error('Error decoding image', e);
+ this.failed = true;
+ }
+ }
+
+
+ public get ready() {
+ return this.bitmap !== null;
+ }
+
+ public getBitmap(_targetWidth: number, _targetHeight: number): ImageBitmap | null {
+ // TODO: mipmap — pick the LOD closest to targetWidth × targetHeight
+ return this.bitmap;
+ }
+
+ public dispose() {
+ if (this.disposed) return;
+ if (this.bitmap) {
+ this.bitmap.close();
+ this.bitmap = null;
+ }
+ this.disposed = true;
+ }
+}
+
+/**
+ * Image decoder backed by `HTMLImageElement`. Used for formats that
+ * `createImageBitmap` doesn't handle reliably across browsers — most
+ * notably SVG, where we want the browser's native vector renderer.
+ */
+export class ElementImageDecoder implements ImageDecoder {
+ readonly assetId: string;
+ public failed = false;
+ private element: HTMLImageElement | null = null;
+ private url: string | null = null;
+ private disposed = false;
+ private readonly width: number;
+ private readonly height: number;
+
+ public constructor(private readonly asset: ImageAsset) {
+ this.assetId = asset.id;
+ this.width = asset.width;
+ this.height = asset.height;
+ }
+
+ public get ready() {
+ return this.element !== null;
+ }
+
+ public async init() {
+ if (this.disposed) return;
+
+ try {
+ const file = await getAssetFile(this.asset);
+ const url = URL.createObjectURL(file);
+
+ await new Promise<void>((resolve) => {
+ const img = new Image();
+ img.onload = () => {
+ if (this.disposed) {
+ URL.revokeObjectURL(url);
+ return resolve();
+ }
+ // SVGs without explicit width/height attributes report
+ // naturalWidth/Height = 0. Pin to the asset's recorded
+ // dimensions so getScaledImageProps gets a real aspect.
+ img.width = this.width;
+ img.height = this.height;
+ this.element = img;
+ this.url = url;
+ resolve();
+ };
+ img.onerror = () => {
+ URL.revokeObjectURL(url);
+ this.failed = true;
+ resolve();
+ };
+ img.src = url;
+ })
+ } catch (e) {
+ console.error('Error decoding image', e);
+ this.failed = true;
+ }
+ }
+
+ public getBitmap(_targetWidth: number, _targetHeight: number): HTMLImageElement | null {
+ return this.element;
+ }
+
+ public dispose() {
+ if (this.disposed) return;
+ if (this.url) {
+ URL.revokeObjectURL(this.url);
+ this.url = null;
+ }
+ this.element = null;
+ this.disposed = true;
+ }
+}
+
+function createImageDecoder(asset: ImageAsset): ImageDecoder {
+ if (asset.mimeType === 'image/svg+xml') {
+ return new ElementImageDecoder(asset);
+ }
+ return new BitmapImageDecoder(asset);
+}
+
+type DecoderCacheEntry = {
+ decoder: ImageDecoder;
+ initPromise: Promise<void>;
+ refs: number;
+};
+
+/** One decoded bitmap per asset id, scoped per world. */
+type ImageDecoderCache = Map<string, DecoderCacheEntry>;
+
+const decoderCaches = new WeakMap<World, ImageDecoderCache>();
+
+function getDecoderCache(world: World): ImageDecoderCache {
+ let cache = decoderCaches.get(world);
+ if (!cache) {
+ cache = new Map();
+ decoderCaches.set(world, cache);
+ }
+ return cache;
+}
+
+/**
+ * Per-entity handle onto a shared, refcounted decoder. Duplicated entities
+ * pointing at the same asset share one decode; disposing a handle releases
+ * one reference and the underlying decoder is disposed with the last one.
+ */
+class SharedImageDecoder implements ImageDecoder {
+ private released = false;
+
+ public constructor(
+ private readonly entry: DecoderCacheEntry,
+ private readonly cache: ImageDecoderCache,
+ ) { }
+
+ public get assetId() {
+ return this.entry.decoder.assetId;
+ }
+ public get failed() {
+ return this.entry.decoder.failed;
+ }
+ public get ready() {
+ return this.entry.decoder.ready;
+ }
+
+ public init(): Promise<void> {
+ return this.entry.initPromise;
+ }
+
+ public getBitmap(targetWidth: number, targetHeight: number): DecodedImage | null {
+ return this.entry.decoder.getBitmap(targetWidth, targetHeight);
+ }
+
+ public dispose() {
+ if (this.released) return;
+ this.released = true;
+ this.entry.refs--;
+ if (this.entry.refs === 0) {
+ this.cache.delete(this.entry.decoder.assetId);
+ this.entry.decoder.dispose();
+ }
+ }
+}
+
+type ResolvedImageDecoder = {
+ decoder: ImageDecoder;
+ initPromise: Promise<void> | null;
+}
+
+export function resolveImageDecoder(world: World, entity: Entity): ResolvedImageDecoder | null {
+ const assetId = entity.get(AssetId)?.value;
+ if (!assetId) return null;
+
+ const existing = entity.get(ImageDecoderHandle);
+ if (existing && existing.assetId === assetId) {
+ return {
+ decoder: existing,
+ initPromise: null,
+ };
+ }
+
+ // Asset changed: release the old decoder and acquire one for the new asset.
+ if (existing) existing.dispose();
+
+ const asset = world.get(Assets)?.get(assetId);
+ if (!asset || asset.type !== 'IMAGE') return null;
+
+ const cache = getDecoderCache(world);
+ let entry = cache.get(assetId);
+ if (!entry) {
+ const decoder = createImageDecoder(asset);
+ entry = { decoder, initPromise: decoder.init(), refs: 0 };
+ cache.set(assetId, entry);
+ }
+ entry.refs++;
+
+ const handle = new SharedImageDecoder(entry, cache);
+ entity.add(ImageDecoderHandle);
+ entity.set(ImageDecoderHandle, handle);
+ return {
+ decoder: handle,
+ initPromise: entry.initPromise,
+ };
+}
diff --git a/packages/runtime/src/media/index.ts b/packages/runtime/src/media/index.ts
index 06882413..86367259 100644
--- a/packages/runtime/src/media/index.ts
+++ b/packages/runtime/src/media/index.ts
@@ -1 +1,18 @@
-// Media decoding: video, audio, image, sequence, caption/, html, shader, surface, frame-cache, time-stretcher (moves wholesale from engine/decoders/).
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export * from './audio';
+export * from './audio-bus';
+export * from './audio-peaks';
+export * from './caption';
+export * from './dispose';
+export * from './frame-cache';
+export * from './html';
+export * from './image';
+export * from './keyframe-index';
+export * from './sequence';
+export * from './shader';
+export * from './surface';
+export * from './time-stretcher';
+export * from './video';
diff --git a/packages/runtime/src/media/keyframe-index.ts b/packages/runtime/src/media/keyframe-index.ts
new file mode 100644
index 00000000..d046fa5f
--- /dev/null
+++ b/packages/runtime/src/media/keyframe-index.ts
@@ -0,0 +1,109 @@
+/**
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { EncodedPacketSink } from 'mediabunny';
+
+import type { InputVideoTrack } from 'mediabunny';
+
+/**
+ * Key packets walked between two yields to the event loop. Keeps an all-intra
+ * track (every packet is a keyframe) from starving the render loop.
+ */
+const YIELD_INTERVAL = 512;
+
+const indexCache = new Map<string, KeyframeIndex>();
+
+export function clearKeyframeIndexCache() {
+ indexCache.clear();
+}
+
+/**
+ * Returns the shared keyframe index for an asset, starting the walk on first
+ * access. Cheap to call repeatedly — one index is built per asset, not per decoder.
+ */
+export function getKeyframeIndex(assetId: string, track: InputVideoTrack): KeyframeIndex {
+ let index = indexCache.get(assetId);
+
+ if (!index) {
+ index = new KeyframeIndex(track);
+ indexCache.set(assetId, index);
+ }
+
+ return index;
+}
+
+/**
+ * Ascending presentation timestamps of every key packet in a video track, walked
+ * ahead of time so seeking resolves its seed keyframe with a binary search
+ * instead of a container lookup.
+ */
+export class KeyframeIndex {
+ private timestamps: number[] = [];
+ private complete = false;
+
+ public constructor(track: InputVideoTrack) {
+ this.build(track);
+ }
+
+ private async build(track: InputVideoTrack) {
+ try {
+ const sink = new EncodedPacketSink(track);
+
+ let packet = await sink.getFirstKeyPacket({ metadataOnly: true });
+ let walked = 0;
+
+ while (packet) {
+ this.timestamps.push(packet.timestamp);
+
+ // `getNextKeyPacket` resolves without I/O once the sample table is parsed,
+ // so awaiting it only queues a microtask and never lets the browser paint.
+ // A timer is a macrotask, which does.
+ if (++walked % YIELD_INTERVAL === 0) {
+ await new Promise((resolve) => setTimeout(resolve));
+ }
+
+ packet = await sink.getNextKeyPacket(packet, { metadataOnly: true });
+ }
+
+ this.complete = true;
+ } catch (e) {
+ // A disposed input (project closed, permission revoked) lands here; the
+ // index simply stays incomplete and callers fall back to a container lookup.
+ console.error('Error building keyframe index', e);
+ }
+ }
+
+ /**
+ * Timestamp of the last keyframe at or before `seconds`, or `null` when the
+ * index cannot answer: `seconds` precedes the first keyframe, or the walk has
+ * not reached that far yet and a closer keyframe may still show up.
+ */
+ public floor(seconds: number): number | null {
+ const timestamps = this.timestamps;
+ const length = timestamps.length;
+
+ if (length === 0 || seconds < timestamps[0]) {
+ return null;
+ }
+ if (!this.complete && seconds > timestamps[length - 1]) {
+ return null;
+ }
+
+ let low = 0;
+ let high = length - 1;
+
+ while (low < high) {
+ const mid = (low + high + 1) >> 1;
+ if (timestamps[mid] <= seconds) {
+ low = mid;
+ } else {
+ high = mid - 1;
+ }
+ }
+
+ return timestamps[low];
+ }
+}
diff --git a/packages/runtime/src/media/sequence.ts b/packages/runtime/src/media/sequence.ts
new file mode 100644
index 00000000..a646a475
--- /dev/null
+++ b/packages/runtime/src/media/sequence.ts
@@ -0,0 +1,266 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { AssetId, SequenceDecoderHandle, Assets } from '../traits';
+import { assert } from '../utils/assert';
+import { FrameCache } from './frame-cache';
+
+import type { Entity, World } from 'koota';
+import type { SequenceAsset } from '../assets/types';
+
+const FRAME_EXTENSIONS = /\.(png|jpe?g|webp|avif|bmp|gif)$/i;
+
+export type SequenceBufferMode = 'discarded' | 'idle' | 'alive';
+const IDLE_TIMEOUT_MS = 10_000;
+
+export class SequenceDecoder {
+ public currentFrame: number = 0;
+ public errored = false;
+ public disposed = false;
+ public hasCache = true;
+ public asset: SequenceAsset;
+ public mode: SequenceBufferMode = 'alive';
+ public initialized: Promise<void>;
+
+ private frames: File[] = [];
+ private lastRenderedFrame: number = -1;
+ private seekGeneration = 0;
+ private idleTimer: ReturnType<typeof setTimeout> | null = null;
+ private seekLock: Promise<void> = Promise.resolve();
+
+ public readonly cache = new FrameCache({ pixels: 720 * 720, count: 49 });
+ public readonly canvas = new OffscreenCanvas(0, 0);
+ public readonly ctx = this.canvas.getContext('2d')!;
+
+ public constructor(asset: SequenceAsset, hasCache: boolean) {
+ this.hasCache = hasCache;
+ this.asset = asset;
+ this.initialized = this.initialize();
+ }
+
+ private async initialize() {
+ try {
+ const entries: { name: string; getFile: () => Promise<File> }[] = [];
+
+ for await (const [name, entry] of this.asset.directoryHandle.entries()) {
+ if (entry.kind !== 'file' || typeof entry.getFile !== 'function') continue;
+ if (!FRAME_EXTENSIONS.test(name)) continue;
+ entries.push({ name, getFile: entry.getFile.bind(entry) });
+ }
+
+ // Natural-order sort so frame_2 comes before frame_10.
+ entries.sort((a, b) => a.name.localeCompare(b.name, undefined, { numeric: true }));
+ this.frames = await Promise.all(entries.map(e => e.getFile()));
+ assert(this.frames.length > 0, 'No frames found');
+ } catch {
+ this.errored = true;
+ }
+ }
+
+ private updateBitmap(frameIndex: number) {
+ if (frameIndex === this.lastRenderedFrame) return;
+ const tile = this.cache.findTile(frameIndex);
+ if (!tile) return;
+ const ctx = this.ctx;
+
+ if (this.canvas.width !== tile.width || this.canvas.height !== tile.height) {
+ this.canvas.width = tile.width;
+ this.canvas.height = tile.height;
+ ctx.imageSmoothingEnabled = false;
+ }
+
+ ctx.clearRect(0, 0, tile.width, tile.height);
+ ctx.drawImage(
+ this.cache.atlas,
+ tile.x,
+ tile.y,
+ tile.width,
+ tile.height,
+ 0,
+ 0,
+ tile.width,
+ tile.height,
+ );
+
+ this.lastRenderedFrame = frameIndex;
+ }
+
+ public toBitmap() {
+ if (this.canvas.width === 0 || this.canvas.height === 0) {
+ return null;
+ }
+
+ return this.canvas;
+ }
+
+ private computeWindow(targetFrame: number, forward: boolean): [number, number] {
+ const span = this.cache.config.count - 2;
+ const lastFrameIndex = this.frames.length - 1;
+
+ // Whole video fits in the cache — keep all of it.
+ if (lastFrameIndex <= span) {
+ return [0, lastFrameIndex];
+ }
+
+ const ahead = Math.round((span * 2) / 3);
+ const behind = span - ahead;
+
+ let left = targetFrame - (forward ? behind : ahead);
+ let right = targetFrame + (forward ? ahead : behind);
+
+ // Push budget that falls outside the boundaries onto the other side.
+ if (left < 0) {
+ right -= left;
+ left = 0;
+ }
+ if (right > lastFrameIndex) {
+ left -= right - lastFrameIndex;
+ right = lastFrameIndex;
+ }
+
+ return [Math.max(0, left), Math.min(lastFrameIndex, right)];
+ }
+
+ private async updateBitmapAsync(frame: number) {
+ try {
+ await this.initialized;
+ const file = this.frames[frame];
+ if (file === undefined) return;
+ const bitmap = await createImageBitmap(file);
+
+ if (this.canvas.width !== bitmap.width || this.canvas.height !== bitmap.height) {
+ this.canvas.width = bitmap.width;
+ this.canvas.height = bitmap.height;
+ this.ctx.imageSmoothingEnabled = false;
+ }
+
+ this.ctx.clearRect(0, 0, bitmap.width, bitmap.height);
+ this.ctx.drawImage(bitmap, 0, 0, bitmap.width, bitmap.height);
+ bitmap.close();
+ } catch { /** ignore */ }
+ }
+
+ public async seekTo(frame: number, frameRate: number) {
+ const targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);
+
+ // Path without cache
+ if (!this.hasCache) {
+ return this.updateBitmapAsync(targetFrame);
+ }
+
+ this.updateBitmap(targetFrame);
+
+ const isCurrentFrame = targetFrame === this.currentFrame;
+ const isEmpty = this.frames.length === 0;
+ const isIdle = this.mode === 'idle' && isCurrentFrame;
+ const isCached = this.cache.has(targetFrame) && isCurrentFrame;
+ const errored = this.errored;
+
+ if (isEmpty || isCached || isIdle || errored) {
+ return;
+ }
+
+ this.touch();
+
+ const generation = ++this.seekGeneration;
+
+ // When the cache does not have the target frame, forward must be true (latency optimization)
+ const forward = this.cache.has(targetFrame) ? targetFrame >= this.currentFrame : true;
+ this.currentFrame = targetFrame;
+
+ const [left, right] = this.computeWindow(targetFrame, forward);
+ this.cache.leftFrameIndex = left;
+ this.cache.rightFrameIndex = right;
+
+ let run: Promise<void>
+ if (forward) {
+ run = this.seekLock.then(() => this.fillCache([targetFrame, this.cache.rightFrameIndex], generation));
+ } else {
+ // +1 to include the target frame
+ run = this.seekLock.then(() => this.fillCache([this.cache.leftFrameIndex, targetFrame], generation));
+ }
+
+ // Run after the previous seek finishes — never concurrently.
+ this.seekLock = run.catch(() => { });
+ }
+
+ private async fillCache(range: [number, number], generation: number): Promise<void> {
+ if (generation !== this.seekGeneration || range[0] > range[1]) return;
+
+ for (let index = range[0]; index <= range[1]; index++) {
+ const file = this.frames[index];
+ if (file === undefined || this.cache.has(index)) continue;
+
+ try {
+ const bitmap = await createImageBitmap(file);
+ this.cache.insert(bitmap, index);
+ bitmap.close();
+ this.updateBitmap(this.currentFrame);
+ } catch { /** ignore */ }
+
+ if (generation !== this.seekGeneration) break;
+ }
+ }
+
+ private touch() {
+ this.mode = 'alive';
+ if (this.idleTimer !== null) {
+ clearTimeout(this.idleTimer);
+ }
+ this.idleTimer = setTimeout(() => this.idle(), IDLE_TIMEOUT_MS);
+ }
+
+ public idle() {
+ if (this.mode !== 'alive') return;
+ this.mode = 'idle';
+
+ if (this.idleTimer !== null) {
+ clearTimeout(this.idleTimer);
+ this.idleTimer = null;
+ }
+
+ this.cache.dispose();
+ }
+
+
+ public dispose() {
+ if (this.mode === 'discarded') return;
+ this.mode = 'discarded';
+
+ if (this.idleTimer !== null) {
+ clearTimeout(this.idleTimer);
+ this.idleTimer = null;
+ }
+
+ this.cache.dispose();
+
+ // Release the display canvas backing store.
+ this.canvas.width = 0;
+ this.canvas.height = 0;
+ this.lastRenderedFrame = -1;
+ }
+}
+
+export function resolveSequenceDecoder(world: World, entity: Entity, hasCache = true): SequenceDecoder | null {
+ const assetId = entity.get(AssetId)?.value;
+ if (!assetId) return null;
+
+ const asset = world.get(Assets)?.get(assetId);
+ if (!asset || asset.type !== 'SEQUENCE') return null;
+
+ // Identity check (not just id) so decoders rebuild when asset metadata —
+ // e.g. frameRate — changes via `assets.save({ ...asset, frameRate: ... })`.
+ const existing = entity.get(SequenceDecoderHandle);
+ if (existing && existing.asset === asset) {
+ existing.hasCache = hasCache;
+ return existing;
+ }
+
+ existing?.dispose();
+
+ const decoder = new SequenceDecoder(asset, hasCache);
+ entity.add(SequenceDecoderHandle);
+ entity.set(SequenceDecoderHandle, decoder);
+ return decoder;
+}
diff --git a/packages/runtime/src/media/shader.ts b/packages/runtime/src/media/shader.ts
new file mode 100644
index 00000000..3fe637ab
--- /dev/null
+++ b/packages/runtime/src/media/shader.ts
@@ -0,0 +1,373 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Shader, ShaderHostHandle } from "../traits";
+import { parseColor } from "../utils/color";
+
+import type { Entity, World } from "koota";
+
+type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+type UniformValue = number | number[] | string;
+
+/**
+ * Runtime pipeline for one shader paint: compiles the paint's fragment-stage
+ * WGSL against the engine prelude, samples the media paint directly below the
+ * shader in the fill stack into a texture, runs the pipeline into an
+ * offscreen WebGPU canvas, and hands the result back to the Canvas2D
+ * compositor via `drawImage`. Created lazily by `resolveShaderHost`, exactly
+ * like the media decoders.
+ *
+ * The authored contract (see reference/jsx/shader-paint.md): the user writes
+ * `@fragment fn main(@location(0) uv: vec2f) -> @location(0) vec4f` with `uv`
+ * in box space, samples the media through `sampleSource(uv)` (which honors
+ * the media paint's `objectFit` placement), and declares parameters as
+ * `@group(1)` uniforms whose values arrive by name from the `uniforms` prop.
+ */
+
+// One device for every shader paint in the page; pipelines are per-host.
+let devicePromise: Promise<GPUDevice | null> | null = null;
+
+function sharedDevice(): Promise<GPUDevice | null> {
+ devicePromise ??= (async () => {
+ const adapter = await navigator.gpu?.requestAdapter();
+ if (!adapter) {
+ console.error("[shader-paint] WebGPU is not available; shader paints render as passthrough");
+ return null;
+ }
+ return await adapter.requestDevice();
+ })();
+ return devicePromise;
+}
+
+const PRELUDE = /* wgsl */ `
+struct Globals {
+ resolution: vec2f,
+ time: f32,
+ _pad: f32,
+ fitOffset: vec2f,
+ fitSize: vec2f,
+}
+
+@group(0) @binding(0) var source: texture_2d<f32>;
+@group(0) @binding(1) var sourceSampler: sampler;
+@group(0) @binding(2) var<uniform> globals: Globals;
+
+struct VSOut {
+ @builtin(position) position: vec4f,
+ @location(0) uv: vec2f,
+}
+
+@vertex
+fn vs(@builtin(vertex_index) i: u32) -> VSOut {
+ var corners = array<vec2f, 3>(vec2f(-1.0, -1.0), vec2f(3.0, -1.0), vec2f(-1.0, 3.0));
+ var out: VSOut;
+ out.position = vec4f(corners[i], 0.0, 1.0);
+ out.uv = vec2f((corners[i].x + 1.0) * 0.5, 1.0 - (corners[i].y + 1.0) * 0.5);
+ return out;
+}
+
+// Samples the media in box space: uv spans the parent's box, the media sits
+// within it per objectFit, and letterbox areas return transparent black.
+fn sampleSource(uv: vec2f) -> vec4f {
+ let suv = (uv - globals.fitOffset) / globals.fitSize;
+ let inside = all(suv >= vec2f(0.0)) && all(suv <= vec2f(1.0));
+ let color = textureSampleLevel(source, sourceSampler, clamp(suv, vec2f(0.0), vec2f(1.0)), 0.0);
+ return select(vec4f(0.0), color, inside);
+}
+`;
+
+const UNIFORM_COMPONENTS: Record<string, number> = {
+ f32: 1,
+ vec2f: 2, "vec2<f32>": 2,
+ vec3f: 3, "vec3<f32>": 3,
+ vec4f: 4, "vec4<f32>": 4,
+};
+
+const UNIFORM_RE = /@group\(1\)\s*@binding\((\d+)\)\s*var\s*<\s*uniform\s*>\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*([A-Za-z0-9_]+(?:<[A-Za-z0-9_]+>)?)/g;
+
+type UniformSlot = {
+ name: string;
+ binding: number;
+ components: number;
+ buffer: GPUBuffer;
+};
+
+type Pipeline = {
+ device: GPUDevice;
+ pipeline: GPURenderPipeline;
+ sampler: GPUSampler;
+ globals: GPUBuffer;
+ slots: UniformSlot[];
+ group1: GPUBindGroup;
+ group0Layout: GPUBindGroupLayout;
+};
+
+export class ShaderHost {
+ private canvas = new OffscreenCanvas(1, 1);
+ private context: GPUCanvasContext | null = null;
+ private gpu: Pipeline | null = null;
+ private sourceTexture: GPUTexture | null = null;
+ private group0: GPUBindGroup | null = null;
+ private code = "";
+ private generation = 0;
+ private compilePromise: Promise<void> | null = null;
+ private colorCache = new Map<string, number | null>();
+ private disposed = false;
+
+ /** Compiles when the code differs from the current pipeline's. */
+ public setCode(code: string): void {
+ if (this.disposed || code === this.code) return;
+ this.code = code;
+ const gen = ++this.generation;
+ this.compilePromise = this.compile(code, gen).catch((error) => {
+ if (gen === this.generation) this.gpu = null;
+ console.error("[shader-paint]", error);
+ });
+ }
+
+ /** Resolves once the current code has compiled (or failed). */
+ public whenReady(): Promise<void> | null {
+ return this.compilePromise;
+ }
+
+ /** True when a pipeline exists — callers fall back to drawing the media directly otherwise. */
+ public get ready(): boolean {
+ return this.gpu !== null && !this.disposed;
+ }
+
+ private async compile(code: string, gen: number): Promise<void> {
+ const device = await sharedDevice();
+ if (!device || this.disposed) return;
+
+ const module = device.createShaderModule({ code: PRELUDE + code });
+ const info = await module.getCompilationInfo();
+ if (gen !== this.generation) return;
+
+ const preludeLines = PRELUDE.split("\n").length;
+ const errors = info.messages.filter((m) => m.type === "error");
+ if (errors.length > 0) {
+ this.gpu = null;
+ const details = errors
+ .map((m) => `${Math.max(1, m.lineNum - preludeLines + 1)}:${m.linePos} ${m.message}`)
+ .join("\n");
+ throw new Error(`WGSL compile failed:\n${details}`);
+ }
+
+ const group0Layout = device.createBindGroupLayout({
+ entries: [
+ { binding: 0, visibility: GPUShaderStage.FRAGMENT, texture: { sampleType: "float" } },
+ { binding: 1, visibility: GPUShaderStage.FRAGMENT, sampler: { type: "filtering" } },
+ { binding: 2, visibility: GPUShaderStage.FRAGMENT, buffer: { type: "uniform" } },
+ ],
+ });
+
+ // Declared @group(1) uniforms get one small buffer each, matched to the
+ // `uniforms` record by name. An explicit layout keeps declared-but-unused
+ // bindings valid (an "auto" layout would strip them).
+ const slots: UniformSlot[] = [];
+ for (const match of code.matchAll(UNIFORM_RE)) {
+ const components = UNIFORM_COMPONENTS[match[3]];
+ if (components === undefined) {
+ console.warn(`[shader-paint] uniform \`${match[2]}\` has unsupported type ${match[3]} (use f32/vec2f/vec3f/vec4f)`);
+ continue;
+ }
+ slots.push({
+ name: match[2],
+ binding: Number(match[1]),
+ components,
+ buffer: device.createBuffer({
+ size: components * 4,
+ usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+ }),
+ });
+ }
+
+ const group1Layout = device.createBindGroupLayout({
+ entries: slots.map((slot) => ({
+ binding: slot.binding,
+ visibility: GPUShaderStage.FRAGMENT,
+ buffer: { type: "uniform" as const },
+ })),
+ });
+
+ const pipeline = device.createRenderPipeline({
+ layout: device.createPipelineLayout({ bindGroupLayouts: [group0Layout, group1Layout] }),
+ vertex: { module, entryPoint: "vs" },
+ fragment: {
+ module,
+ entryPoint: "main",
+ targets: [{ format: navigator.gpu.getPreferredCanvasFormat() }],
+ },
+ });
+ if (gen !== this.generation || this.disposed) return;
+
+ this.gpu = {
+ device,
+ pipeline,
+ sampler: device.createSampler({ magFilter: "linear", minFilter: "linear" }),
+ globals: device.createBuffer({ size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST }),
+ slots,
+ group1: device.createBindGroup({
+ layout: group1Layout,
+ entries: slots.map((slot) => ({ binding: slot.binding, resource: { buffer: slot.buffer } })),
+ }),
+ group0Layout,
+ };
+ // The source texture and its bind group belong to the old pipeline's layout.
+ this.sourceTexture?.destroy();
+ this.sourceTexture = null;
+ this.group0 = null;
+ }
+
+ private writeUniforms(uniforms: Record<string, UniformValue> | null): void {
+ const gpu = this.gpu!;
+ for (const slot of gpu.slots) {
+ const value = uniforms?.[slot.name];
+ if (value === undefined) continue;
+ const data = new Float32Array(slot.components);
+ if (typeof value === "number") {
+ data[0] = value;
+ } else if (Array.isArray(value)) {
+ for (let i = 0; i < Math.min(slot.components, value.length); i++) data[i] = value[i];
+ } else {
+ if (!this.colorCache.has(value)) this.colorCache.set(value, parseColor(value));
+ const color = this.colorCache.get(value);
+ if (color == null) continue;
+ data[0] = ((color >> 16) & 0xff) / 255;
+ if (slot.components >= 2) data[1] = ((color >> 8) & 0xff) / 255;
+ if (slot.components >= 3) data[2] = (color & 0xff) / 255;
+ if (slot.components >= 4) data[3] = 1;
+ }
+ gpu.device.queue.writeBuffer(slot.buffer, 0, data);
+ }
+ }
+
+ /**
+ * Runs the shader over `source` and draws the result into the parent's
+ * box. `fit` is the media's drawImage placement within the box (from
+ * getScaledImageProps); `time` is the parent's local time in seconds.
+ * A null `source` runs the shader procedurally: the bound texture stays
+ * transparent black, so `sampleSource` returns vec4f(0) everywhere.
+ * Returns false when no pipeline is ready — the caller then draws the
+ * media directly instead.
+ */
+ public draw(
+ ctx: Ctx2D,
+ width: number,
+ height: number,
+ source: GPUCopyExternalImageSource | null,
+ sourceWidth: number,
+ sourceHeight: number,
+ fit: [dx: number, dy: number, sw: number, sh: number],
+ time: number,
+ uniforms: Record<string, UniformValue> | null,
+ ): boolean {
+ const gpu = this.gpu;
+ if (!gpu || this.disposed) return false;
+
+ const w = Math.max(1, Math.round(width));
+ const h = Math.max(1, Math.round(height));
+ if (this.canvas.width !== w || this.canvas.height !== h) {
+ this.canvas.width = w;
+ this.canvas.height = h;
+ }
+ if (this.context === null) {
+ this.context = this.canvas.getContext("webgpu");
+ if (!this.context) return false;
+ this.context.configure({
+ device: gpu.device,
+ format: navigator.gpu.getPreferredCanvasFormat(),
+ alphaMode: "premultiplied",
+ });
+ }
+
+ const sw = source ? Math.max(1, Math.round(sourceWidth)) : 1;
+ const sh = source ? Math.max(1, Math.round(sourceHeight)) : 1;
+ if (!this.sourceTexture || this.sourceTexture.width !== sw || this.sourceTexture.height !== sh || !this.group0) {
+ this.sourceTexture?.destroy();
+ this.sourceTexture = gpu.device.createTexture({
+ size: [sw, sh],
+ format: "rgba8unorm",
+ usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.RENDER_ATTACHMENT,
+ });
+ this.group0 = gpu.device.createBindGroup({
+ layout: gpu.group0Layout,
+ entries: [
+ { binding: 0, resource: this.sourceTexture.createView() },
+ { binding: 1, resource: gpu.sampler },
+ { binding: 2, resource: { buffer: gpu.globals } },
+ ],
+ });
+ }
+
+ if (source) {
+ gpu.device.queue.copyExternalImageToTexture(
+ { source },
+ { texture: this.sourceTexture, premultipliedAlpha: true },
+ [sw, sh],
+ );
+ }
+
+ const [dx, dy, dw, dh] = fit;
+ gpu.device.queue.writeBuffer(gpu.globals, 0, new Float32Array([
+ w, h, time, 0,
+ dx / w, dy / h, dw / w, dh / h,
+ ]));
+ this.writeUniforms(uniforms);
+
+ const encoder = gpu.device.createCommandEncoder();
+ const pass = encoder.beginRenderPass({
+ colorAttachments: [{
+ view: this.context.getCurrentTexture().createView(),
+ clearValue: { r: 0, g: 0, b: 0, a: 0 },
+ loadOp: "clear",
+ storeOp: "store",
+ }],
+ });
+ pass.setPipeline(gpu.pipeline);
+ pass.setBindGroup(0, this.group0);
+ pass.setBindGroup(1, gpu.group1);
+ pass.draw(3);
+ pass.end();
+ gpu.device.queue.submit([encoder.finish()]);
+
+ ctx.drawImage(this.canvas, 0, 0, width, height);
+ return true;
+ }
+
+ public dispose(): void {
+ if (this.disposed) return;
+ this.disposed = true;
+ this.generation++;
+ this.sourceTexture?.destroy();
+ this.sourceTexture = null;
+ if (this.gpu) {
+ this.gpu.globals.destroy();
+ for (const slot of this.gpu.slots) slot.buffer.destroy();
+ this.gpu = null;
+ }
+ this.context?.unconfigure();
+ this.canvas.width = 0;
+ this.canvas.height = 0;
+ }
+}
+
+/**
+ * The shader paint's host, created lazily like the media decoders and kept in
+ * sync with the persisted `Shader.code` component. Returns null until the
+ * paint carries code.
+ */
+export function resolveShaderHost(_world: World, entity: Entity): ShaderHost | null {
+ const code = entity.get(Shader)?.code;
+ if (!code) return null;
+
+ let host = entity.get(ShaderHostHandle) ?? null;
+ if (!host) {
+ host = new ShaderHost();
+ entity.add(ShaderHostHandle);
+ entity.set(ShaderHostHandle, host);
+ }
+ host.setCode(code);
+ return host;
+}
diff --git a/packages/runtime/src/media/surface.ts b/packages/runtime/src/media/surface.ts
new file mode 100644
index 00000000..c4d5a803
--- /dev/null
+++ b/packages/runtime/src/media/surface.ts
@@ -0,0 +1,39 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+
+/**
+ * Backing store for one surface paint: a detached canvas handed to the
+ * mount's `ref` callback, which draws into it with any context type (2d,
+ * webgl, webgpu). The engine samples the bitmap every frame and stretches it
+ * into the parent geometry's box. After the initial box-sized allocation the
+ * bitmap belongs to user code — external renderers (e.g. three.js via
+ * `renderer.setSize`) resize it through their own APIs, so the engine never
+ * touches it again.
+ */
+export class SurfaceHost {
+ public readonly canvas = document.createElement('canvas');
+ private disposed = false;
+
+ /** Initial bitmap allocation — called once at materialization, before the ref runs. */
+ public setSize(width: number, height: number): void {
+ this.canvas.width = Math.max(1, Math.round(width));
+ this.canvas.height = Math.max(1, Math.round(height));
+ }
+
+ public draw(ctx: Ctx2D, width: number, height: number): void {
+ if (this.disposed || this.canvas.width === 0 || this.canvas.height === 0) return;
+ ctx.drawImage(this.canvas, 0, 0, width, height);
+ }
+
+ public dispose(): void {
+ if (this.disposed) return;
+ this.disposed = true;
+ // Release the backing store; context teardown (e.g. a WebGL renderer)
+ // is the mount's own onCleanup concern.
+ this.canvas.width = 0;
+ this.canvas.height = 0;
+ }
+}
diff --git a/packages/runtime/src/media/time-stretcher.ts b/packages/runtime/src/media/time-stretcher.ts
new file mode 100644
index 00000000..4af25cf5
--- /dev/null
+++ b/packages/runtime/src/media/time-stretcher.ts
@@ -0,0 +1,260 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+
+/**
+ * Utility class for performing time stretching on a multi-channel audio signal. The input audio will be stretched by
+ * a configurable factor without changing its pitch.
+ *
+ * Internally, this uses a WSOLA-like algorithm.
+ */
+export class TimeStretcher {
+ factor: number; // This value can be changed at runtime
+
+ private numberOfChannels: number;
+ private synthesisHopSize: number;
+ private windowSize: number;
+ private overlapSize: number;
+ private tolerance: number;
+ private buffers: Float32Array[];
+ private bufferEndIndex: number;
+ private outputBuffers: Float32Array[];
+ private nextOutputBufferShiftAmount: number;
+ private outputBufferLength: number;
+ private hasDoneOutput: boolean;
+ private finalized: boolean;
+ private blendValues: number[];
+
+ public constructor(numberOfChannels: number, sampleRate: number, factor: number) {
+ this.numberOfChannels = numberOfChannels;
+ this.synthesisHopSize = Math.floor((512 * sampleRate) / 48000);
+ this.windowSize = 2 * this.synthesisHopSize;
+ this.overlapSize = this.synthesisHopSize;
+ this.tolerance = this.synthesisHopSize;
+ this.buffers = [];
+ this.bufferEndIndex = this.tolerance;
+ this.factor = factor;
+ this.outputBuffers = [];
+ this.nextOutputBufferShiftAmount = 0;
+ this.outputBufferLength = 2 ** 16;
+ this.hasDoneOutput = false;
+ this.finalized = false;
+
+ for (let i = 0; i < numberOfChannels; i++) {
+ this.buffers.push(new Float32Array(2 ** 16));
+ this.outputBuffers.push(new Float32Array(this.outputBufferLength));
+ }
+
+ this.blendValues = [];
+ for (let i = 0; i < this.overlapSize; i++) {
+ this.blendValues[i] = 0.5 * (1 - Math.cos((Math.PI * i) / this.overlapSize));
+ }
+ }
+
+ private ensureOutputBufferLength(requiredLength: number): void {
+ if (requiredLength > this.outputBufferLength) {
+ this.outputBufferLength = requiredLength;
+ for (let i = 0; i < this.numberOfChannels; i++) {
+ const largerBuffer = new Float32Array(this.outputBufferLength);
+ largerBuffer.set(this.outputBuffers[i], 0);
+ this.outputBuffers[i] = largerBuffer;
+ }
+ }
+ }
+
+ private clearOutputBuffers(): void {
+ if (this.nextOutputBufferShiftAmount > 0) {
+ for (let i = 0; i < this.numberOfChannels; i++) {
+ const buffer = this.outputBuffers[i];
+ buffer.set(buffer.subarray(this.nextOutputBufferShiftAmount), 0);
+ }
+
+ this.nextOutputBufferShiftAmount = 0;
+ }
+ }
+
+ private synthesizeSegment(
+ i: number,
+ windowSize: number,
+ inputStartPos: number,
+ maxPositiveOffset: number = this.tolerance
+ ): void {
+ const crossCorrelateAllChannels = (k: number): number => {
+ let dot = 0;
+ let normOldTotal = 0;
+ let normNewTotal = 0;
+
+ for (let chan = 0; chan < this.numberOfChannels; chan++) {
+ const inputBuffer = this.buffers[chan];
+ const outputBuffer = this.outputBuffers[chan];
+
+ for (let j = 0; j < this.overlapSize; j++) {
+ const oldValue = outputBuffer[i + j];
+ const newValue = inputBuffer[inputStartPos + k + j];
+
+ dot += oldValue * newValue;
+ normOldTotal += oldValue ** 2;
+ normNewTotal += newValue ** 2;
+ }
+ }
+
+ const ncc = dot / (Math.sqrt(normOldTotal * normNewTotal) || 1e-10);
+ return ncc;
+ };
+
+ let bestCorr = -Infinity;
+ let bestOffset = 0;
+
+ if (!this.hasDoneOutput || this.factor === 1) {
+ bestOffset = 0;
+ } else {
+ let k = -this.tolerance;
+ bestCorr = crossCorrelateAllChannels(k);
+ bestOffset = k;
+
+ const minStepSize = 1;
+ const maxStepSize = 16;
+ let prevCorr = bestCorr;
+
+ while (k < maxPositiveOffset) {
+ const nextK = Math.min(k + 1, maxPositiveOffset);
+ const corr = crossCorrelateAllChannels(nextK);
+
+ const gradient = Math.abs(corr - prevCorr);
+ const adaptiveStep = Math.max(
+ minStepSize,
+ Math.min(maxStepSize, Math.floor(maxStepSize * Math.exp(-gradient * 3)))
+ );
+
+ if (corr > bestCorr) {
+ bestCorr = corr;
+ bestOffset = nextK;
+ }
+
+ prevCorr = corr;
+ k += adaptiveStep;
+ }
+
+ // Fine-tuning phase
+ const fineRange = 8;
+ for (k = bestOffset - fineRange; k <= bestOffset + fineRange; k++) {
+ if (k >= -this.tolerance && k <= maxPositiveOffset) {
+ const corr = crossCorrelateAllChannels(k);
+ if (corr > bestCorr) {
+ bestCorr = corr;
+ bestOffset = k;
+ }
+ }
+ }
+ }
+
+ for (let chan = 0; chan < this.numberOfChannels; chan++) {
+ const inputBuffer = this.buffers[chan];
+ const outputBuffer = this.outputBuffers[chan];
+
+ for (let j = 0; j < windowSize; j++) {
+ const blendValue = j < this.overlapSize ? this.blendValues[j] : 1;
+ outputBuffer[i + j] *= 1 - blendValue;
+ outputBuffer[i + j] += blendValue * inputBuffer[inputStartPos + bestOffset + j];
+ }
+ }
+
+ this.hasDoneOutput = true;
+ }
+
+ public append(newBuffers: Float32Array[]): Float32Array[] | null {
+ if (this.finalized) {
+ throw new Error('Cannot append more buffers after calling finalize.');
+ }
+
+ // Fast path: when factor is 1, pass through directly without any processing
+ if (this.factor === 1) {
+ this.hasDoneOutput = true;
+ return newBuffers;
+ }
+
+ for (let i = 0; i < this.numberOfChannels; i++) {
+ const newBuffer = newBuffers[i];
+ const requiredLength = this.bufferEndIndex + newBuffer.length;
+ let currentLength = this.buffers[i].length;
+
+ while (currentLength < requiredLength) {
+ currentLength *= 2;
+ }
+
+ if (currentLength !== this.buffers[i].length) {
+ const largerBuffer = new Float32Array(currentLength);
+ largerBuffer.set(this.buffers[i], 0);
+ this.buffers[i] = largerBuffer;
+ }
+
+ this.buffers[i].set(newBuffer, this.bufferEndIndex);
+ }
+
+ this.bufferEndIndex += newBuffers[0].length;
+
+ return this.process();
+ }
+
+ private process(): Float32Array[] | null {
+ let synthesisLength = 0;
+ // eslint-disable-next-line no-constant-condition
+ for (synthesisLength; true; synthesisLength += this.synthesisHopSize) {
+ const inputShiftAmount = Math.floor(synthesisLength / this.factor);
+
+ if (inputShiftAmount + this.windowSize + 2 * this.tolerance > this.bufferEndIndex) {
+ synthesisLength -= this.synthesisHopSize;
+ break;
+ }
+ }
+
+ if (synthesisLength <= 0) {
+ return null;
+ }
+
+ const inputShiftAmount = Math.floor(synthesisLength / this.factor);
+ const outputLength = synthesisLength - this.synthesisHopSize + this.windowSize;
+
+ this.clearOutputBuffers();
+ this.ensureOutputBufferLength(outputLength);
+
+ for (let i = 0; i < synthesisLength; i += this.synthesisHopSize) {
+ const inputStartPos = this.tolerance + Math.floor(i / this.factor);
+ this.synthesizeSegment(i, this.windowSize, inputStartPos);
+ }
+
+ // Shift all input buffers
+ for (let chan = 0; chan < this.numberOfChannels; chan++) {
+ const inputBuffer = this.buffers[chan];
+ inputBuffer.set(inputBuffer.subarray(inputShiftAmount, this.bufferEndIndex), 0);
+ }
+
+ this.bufferEndIndex -= inputShiftAmount;
+ this.nextOutputBufferShiftAmount = synthesisLength;
+
+ return this.outputBuffers.map(x => x.subarray(0, synthesisLength));
+ }
+
+ public finalize(): Float32Array[] | null {
+ this.finalized = true;
+
+ // Fast path: when factor is 1, nothing is buffered
+ if (this.factor === 1) {
+ return null;
+ }
+
+ if (this.bufferEndIndex <= this.tolerance) {
+ return null;
+ }
+
+ this.clearOutputBuffers();
+
+ const synthesisLength = this.bufferEndIndex - this.tolerance;
+ this.ensureOutputBufferLength(synthesisLength);
+
+ this.synthesizeSegment(0, synthesisLength, this.tolerance, 0);
+
+ return this.outputBuffers.map(x => x.subarray(0, synthesisLength));
+ }
+}
diff --git a/packages/runtime/src/media/video.ts b/packages/runtime/src/media/video.ts
new file mode 100644
index 00000000..b8aa86bc
--- /dev/null
+++ b/packages/runtime/src/media/video.ts
@@ -0,0 +1,731 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { BlobSource, ALL_FORMATS, Input, InputVideoTrack, EncodedPacketSink, EncodedPacket, CanvasSink, type WrappedCanvas } from 'mediabunny';
+
+import { AssetId, VideoDecoderHandle, Assets, Mode } from '../traits';
+import { assert } from '../utils/assert';
+import { getAssetFile } from '../actions/assets';
+import { FrameCache } from './frame-cache';
+import { getKeyframeIndex } from './keyframe-index';
+
+import type { Entity, World } from 'koota';
+import type { KeyframeIndex } from './keyframe-index';
+import type { VideoAsset } from '../assets/types';
+
+
+export type VideoBufferMode = 'discarded' | 'idle' | 'alive';
+
+/**
+ * Inactivity window after which an `alive` buffer automatically drops to
+ * `idle`, freeing its decoder and frame cache while keeping the display canvas.
+ */
+const IDLE_TIMEOUT_MS = 10_000;
+
+/**
+ * Max extra frames we are willing to decode forward from the live cursor in
+ * order to avoid a keyframe lookup + decoder reseed. Higher = stronger bias.
+ */
+const FORWARD_BIAS_FRAMES = 24;
+
+/**
+ * Frames used to drain the decoder in case of a backward seek to
+ * ensure the target frame is surfaced.
+ */
+const DRAIN_FRAMES = 8;
+
+/**
+ * How far the preview may drift from the requested frame. A neighbour this close
+ * beats holding the last drawn frame while the exact one is still decoding.
+ */
+const DISPLAY_TOLERANCE_FRAMES = 4;
+
+/**
+ * Total pixel budget of the preview frame cache
+ */
+const CACHE_PIXEL_BUDGET = 768 * 432 * 81; // 81 tiles at 768x432
+
+/**
+ * Per-tile pixel cap (~720p) — enough detail for the preview canvas.
+ */
+const MAX_TILE_PIXELS = 1280 * 720;
+
+/**
+ * Two seeks arriving closer together than this are treated as one continuous drag.
+ */
+const SCRUB_EVENT_WINDOW_MS = 250;
+
+/**
+ * Quiet period after the last scrub seek before the exact frame is resolved.
+ */
+const SCRUB_SETTLE_MS = 120;
+
+const MIN_CACHE_COUNT = 30;
+const MAX_CACHE_COUNT = 81;
+
+export class VideoBuffer {
+ public errored = false;
+ public asset: VideoAsset;
+ public firstPacketTimestamp = 0;
+ public packetSink: EncodedPacketSink | null = null;
+ public mode: VideoBufferMode = 'alive';
+ public initialized: Promise<void>;
+
+ public readonly cache: FrameCache;
+ public readonly queue = new VideoDecoderQueue(this.frameCallback.bind(this));
+
+ // user facing display canvas
+ public readonly canvas = new OffscreenCanvas(0, 0);
+ public readonly ctx = this.canvas.getContext('2d')!;
+
+ private currentFrame: number = -1;
+ private isDirty: boolean = true;
+ private keyframes: KeyframeIndex | null = null;
+
+ /**
+ * Frame the preview is aiming to show. Tracks `currentFrame` except while scrubbing,
+ * where it points at the covering keyframe instead of the requested frame.
+ */
+ private displayFrame: number = -1;
+
+ /**
+ * Keyframes a scrub is waiting on. The preview keeps showing the last one that landed
+ * until the next arrives, so a drag never blanks out for the length of a decode.
+ * Holds several at once: decoder output trails submission by a handful of packets.
+ */
+ private readonly pendingScrub = new Set<number>();
+
+ private lastFrameIndex: number = 0;
+ private seekGeneration = 0;
+ private seekLock: Promise<void> = Promise.resolve();
+ private iterator: AsyncGenerator<EncodedPacket, void, unknown> | null = null;
+ private idleTimer: ReturnType<typeof setTimeout> | null = null;
+ private settleTimer: ReturnType<typeof setTimeout> | null = null;
+ private lastSeekAt: number = -Infinity;
+
+ public constructor(asset: VideoAsset) {
+ this.asset = asset;
+
+ const pixels = Math.min(asset.width * asset.height, MAX_TILE_PIXELS);
+ const count = Math.min(MAX_CACHE_COUNT, Math.max(MIN_CACHE_COUNT, Math.floor(CACHE_PIXEL_BUDGET / pixels)));
+ this.cache = new FrameCache({ pixels, count });
+
+ this.initialized = this.initialize();
+ }
+
+ private async initialize() {
+ try {
+ const videoTrack = await getVideoTrack(this.asset);
+
+ assert(videoTrack, 'Video track not found');
+
+ this.keyframes = getKeyframeIndex(this.asset.id, videoTrack);
+
+ await this.queue.init(videoTrack);
+
+ this.cache.rotation = videoTrack.rotation;
+ this.packetSink = new EncodedPacketSink(videoTrack);
+ this.firstPacketTimestamp = Math.max(0, await videoTrack.getFirstTimestamp() ?? 0);
+ this.lastFrameIndex = Math.max(0, Math.round(this.asset.duration * this.asset.frameRate) - 1);
+ this.isDirty = true;
+ } catch (e) {
+ console.error('Error initializing video decoder', e);
+ this.errored = true;
+ }
+ }
+
+ private frameCallback(frame: VideoFrame) {
+ const timestampSeconds = frame.timestamp / 1e6;
+ const frameIndex = this.secondsToFrames(timestampSeconds);
+ this.cache.insert(frame, frameIndex);
+ this.isDirty = true;
+
+ // A scrub keyframe takes over the preview the moment it lands, and only then:
+ // matching on the index keeps frames from an abandoned fill out of the display.
+ if (this.pendingScrub.delete(frameIndex)) {
+ this.displayFrame = frameIndex;
+ }
+ }
+
+ public seekTo(frame: number, frameRate: number): undefined {
+ const targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);
+
+ const isEmpty = !this.packetSink;
+ const isCurrentFrame = targetFrame === this.currentFrame;
+ const isCached = this.cache.has(targetFrame) && isCurrentFrame;
+ const isIdle = this.mode === 'idle' && isCurrentFrame;
+ const errored = this.errored;
+
+ if (isEmpty || isCurrentFrame || isCached || isIdle || errored) {
+ return;
+ }
+
+ const previousFrame = this.currentFrame;
+ const consecutive = performance.now() - this.lastSeekAt < SCRUB_EVENT_WINDOW_MS;
+ this.lastSeekAt = performance.now();
+
+ this.currentFrame = targetFrame;
+ this.isDirty = true;
+
+ this.touch();
+
+ // A run of large jumps is a drag, not a playback step. Walking each GOP to the
+ // exact frame costs more than the gap between pointer events, so every walk gets
+ // cancelled by the next one and nothing ever paints.
+ const jumped = Math.abs(targetFrame - previousFrame) > FORWARD_BIAS_FRAMES;
+
+ if (consecutive && jumped && this.scrubTo(targetFrame)) {
+ return;
+ }
+
+ this.exactSeekTo(targetFrame, previousFrame);
+ }
+
+ /**
+ * Paints the keyframe covering `targetFrame` rather than the frame itself: one packet,
+ * no walk through the GOP. Returns false when the keyframe index cannot place the
+ * target yet, leaving the caller to fall back to an exact seek.
+ */
+ private scrubTo(targetFrame: number): boolean {
+ const keyTimestamp = this.keyframes?.floor(this.framesToSeconds(targetFrame)) ?? null;
+
+ if (keyTimestamp === null) {
+ return false;
+ }
+
+ const keyFrame = this.secondsToFrames(keyTimestamp);
+ this.scheduleSettle();
+
+ // Supersedes any fill still running for a position we have already left.
+ const generation = ++this.seekGeneration;
+
+ // Consecutive positions inside one GOP resolve to a keyframe we already hold.
+ if (this.cache.has(keyFrame)) {
+ this.displayFrame = keyFrame;
+ return true;
+ }
+
+ this.pendingScrub.add(keyFrame);
+
+ this.seekLock = this.seekLock
+ .then(() => this.decodeKeyframe(keyTimestamp, generation))
+ .catch(() => { });
+
+ return true;
+ }
+
+ private async decodeKeyframe(keyTimestamp: number, generation: number): Promise<void> {
+ if (generation !== this.seekGeneration) return;
+
+ const keyPacket = await this.packetSink?.getKeyPacket(keyTimestamp);
+ if (!keyPacket || generation !== this.seekGeneration) return;
+
+ // Seed the iterator here as well, so the settle pass continues forward from this
+ // keyframe instead of reseeding the decoder a second time.
+ await this.iterator?.return();
+ const iterator = this.packetSink?.packets(keyPacket) ?? null;
+ this.iterator = iterator;
+ if (!iterator) return;
+
+ if (!this.queue.isAlive) {
+ this.queue.reseed();
+ }
+
+ const { value: packet } = await iterator.next();
+ if (packet) {
+ await this.queue.decode(packet);
+ }
+ }
+
+ /**
+ * Re-runs the seek for real once the drag stops.
+ */
+ private scheduleSettle() {
+ if (this.settleTimer !== null) {
+ clearTimeout(this.settleTimer);
+ }
+
+ this.settleTimer = setTimeout(() => {
+ this.settleTimer = null;
+ if (this.mode !== 'alive' || this.errored) return;
+ this.exactSeekTo(this.currentFrame, this.currentFrame);
+ }, SCRUB_SETTLE_MS);
+ }
+
+ private exactSeekTo(targetFrame: number, previousFrame: number) {
+ this.displayFrame = targetFrame;
+ this.pendingScrub.clear();
+
+ const forward = this.isBlockedFrame(targetFrame)
+ ? targetFrame >= previousFrame
+ : true;
+
+ const generation = ++this.seekGeneration;
+
+ const [left, right] = this.computeWindow(targetFrame, forward);
+ this.cache.leftFrameIndex = left;
+ this.cache.rightFrameIndex = right;
+
+ let seed = targetFrame;
+ let run: Promise<void>
+ if (forward) {
+ while (this.isBlockedFrame(seed) && seed <= this.cache.rightFrameIndex) {
+ seed++;
+ }
+
+ run = this.seekLock.then(() => this.fillCache([seed, this.cache.rightFrameIndex], generation));
+ } else {
+ while (this.cache.has(seed) && seed >= this.cache.leftFrameIndex) {
+ seed--;
+ }
+
+ // TODO: Find a better solution for DRAIN_FRAMES
+ run = this.seekLock.then(() => this.fillCache([this.cache.leftFrameIndex, seed + DRAIN_FRAMES], generation));
+ }
+
+ // Run after the previous seek finishes — never concurrently.
+ this.seekLock = run.catch(() => { });
+ }
+
+ private framesToSeconds(frames: number) {
+ return Math.max(this.firstPacketTimestamp, (frames / this.asset.frameRate) + this.firstPacketTimestamp);
+ }
+
+ private secondsToFrames(seconds: number) {
+ return Math.max(0, Math.round((seconds - this.firstPacketTimestamp) * this.asset.frameRate));
+ }
+
+ /**
+ * Whether `frame` is already taken care of — decoded into the cache, or still in-flight.
+ */
+ private isBlockedFrame(frame: number) {
+ if (this.cache.has(frame)) return true;
+
+ for (const micros of this.queue.inFlight) {
+ if (this.secondsToFrames(micros / 1e6) === frame) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private computeWindow(targetFrame: number, forward: boolean): [number, number] {
+ const span = this.cache.config.count - 2;
+
+ // Whole video fits in the cache — keep all of it.
+ if (this.lastFrameIndex <= span) {
+ return [0, this.lastFrameIndex];
+ }
+
+ const ahead = Math.round((span * 2) / 3);
+ const behind = span - ahead;
+
+ let left = targetFrame - (forward ? behind : ahead);
+ let right = targetFrame + (forward ? ahead : behind);
+
+ // Push budget that falls outside the boundaries onto the other side.
+ if (left < 0) {
+ right -= left;
+ left = 0;
+ }
+ if (right > this.lastFrameIndex) {
+ left -= right - this.lastFrameIndex;
+ right = this.lastFrameIndex;
+ }
+
+ return [Math.max(0, left), Math.min(this.lastFrameIndex, right)];
+ }
+
+ private async fillCache(range: [number, number], generation: number): Promise<void> {
+ if (generation !== this.seekGeneration || Math.abs(range[1] - range[0]) <= 3) return;
+ // Note: only decode for three or more frames at a time to avoid unnecessary reseeding
+
+ const fromSecs = this.framesToSeconds(range[0]);
+ const untilSecs = this.framesToSeconds(range[1]);
+ const cursor = this.queue.lastSubmitted;
+ const live = !!(this.iterator && this.queue.isAlive && cursor);
+
+ let reuse = live && cursor!.timestamp < untilSecs;
+ let keyTimestamp = this.keyframes?.floor(fromSecs) ?? null;
+
+ // If the cursor lags far behind the range start, skip forward to the nearest
+ // keyframe rather than decoding through the whole gap.
+ const biasSecs = FORWARD_BIAS_FRAMES / this.asset.frameRate;
+ if (reuse && cursor!.timestamp < fromSecs - biasSecs) {
+ keyTimestamp ??= (await this.packetSink?.getKeyPacket(fromSecs))?.timestamp ?? null;
+ if (keyTimestamp !== null && keyTimestamp - cursor!.timestamp > biasSecs) {
+ reuse = false; // jumping to the keyframe
+ }
+ }
+
+ // we cant reuse the iterator, so we need to reseed the decoder
+ // and get create a new iterator
+ if (!reuse) {
+ const keyPacket = (await this.packetSink?.getKeyPacket(keyTimestamp ?? fromSecs)) ?? null;
+ if (!keyPacket) return;
+ await this.iterator?.return();
+ this.iterator = this.packetSink?.packets(keyPacket) ?? null;
+ }
+
+ if (!this.queue.isAlive) {
+ this.queue.reseed();
+ }
+
+ if (generation !== this.seekGeneration || !this.iterator) return;
+
+ while (true) {
+ const { value: packet, done } = await this.iterator.next();
+ if (done || !packet) break;
+
+ await this.queue.decode(packet);
+
+ if (generation !== this.seekGeneration || packet.timestamp >= untilSecs) break;
+ }
+ }
+
+ public toBitmap() {
+ if (this.isDirty && this.displayFrame >= 0) {
+
+ const nearest = this.cache.findNearest(this.displayFrame, DISPLAY_TOLERANCE_FRAMES);
+ const tile = nearest === undefined ? undefined : this.cache.findTile(nearest);
+ const ctx = this.ctx;
+
+ if (tile && (this.canvas.width !== tile.width || this.canvas.height !== tile.height)) {
+ this.canvas.width = tile.width;
+ this.canvas.height = tile.height;
+ this.ctx.imageSmoothingEnabled = false;
+ }
+
+ if (tile) {
+ ctx.drawImage(
+ this.cache.atlas,
+ tile.x,
+ tile.y,
+ tile.width,
+ tile.height,
+ 0,
+ 0,
+ tile.width,
+ tile.height,
+ );
+ this.isDirty = false;
+ }
+ }
+
+ if (this.canvas.width === 0 || this.canvas.height === 0) {
+ return null;
+ }
+
+ return this.canvas;
+ }
+
+ private touch() {
+ this.mode = 'alive';
+ if (this.idleTimer !== null) {
+ clearTimeout(this.idleTimer);
+ }
+ this.idleTimer = setTimeout(() => this.idle(), IDLE_TIMEOUT_MS);
+ }
+
+ public idle() {
+ if (this.mode !== 'alive') return;
+ this.mode = 'idle';
+
+ if (this.idleTimer !== null) {
+ clearTimeout(this.idleTimer);
+ this.idleTimer = null;
+ }
+
+ if (this.settleTimer !== null) {
+ clearTimeout(this.settleTimer);
+ this.settleTimer = null;
+ }
+
+ this.pendingScrub.clear();
+ this.cache.dispose();
+ this.queue.dispose();
+ this.iterator?.return();
+ this.iterator = null;
+ }
+
+ public dispose() {
+ if (this.mode === 'discarded') return;
+ this.mode = 'discarded';
+
+ if (this.idleTimer !== null) {
+ clearTimeout(this.idleTimer);
+ this.idleTimer = null;
+ }
+
+ if (this.settleTimer !== null) {
+ clearTimeout(this.settleTimer);
+ this.settleTimer = null;
+ }
+
+ this.pendingScrub.clear();
+ this.cache.dispose();
+ this.queue.dispose();
+ this.iterator?.return();
+ this.iterator = null;
+
+ // Release the display canvas backing store.
+ this.canvas.width = 0;
+ this.canvas.height = 0;
+ }
+}
+
+class VideoDecoderQueue {
+ private config: VideoDecoderConfig | null = null;
+ private decoder: VideoDecoder | null = null;
+ private resolver: ReturnType<typeof Promise.withResolvers> | null = null;
+ private callback: (frame: VideoFrame) => void;
+ public lastSubmitted: EncodedPacket | null = null;
+ public readonly inFlight = new Set<number>();
+
+ public constructor(callback: (frame: VideoFrame) => void) {
+ this.callback = callback;
+ }
+
+ public get isAlive() {
+ return this.decoder?.state === 'configured';
+ }
+
+ /**
+ * Synchronously discard all in-flight work and re-arm the decoder for a fresh keyframe
+ */
+ public reseed() {
+ if (!this.decoder) return;
+ this.decoder.reset();
+ assert(this.config, 'Decoder config not available');
+ this.decoder.configure(this.config);
+ this.resolver?.resolve(null);
+ this.resolver = null;
+ this.lastSubmitted = null;
+ this.inFlight.clear();
+ }
+
+ private handleOutput(frame: VideoFrame) {
+ this.resolver?.resolve(null);
+ this.inFlight.delete(frame.timestamp);
+
+ for (const micros of this.inFlight) {
+ if (micros < frame.timestamp) {
+ this.inFlight.delete(micros);
+ }
+ }
+
+ this.callback(frame);
+ frame.close();
+ }
+
+ private handleDequeue() {
+ this.resolver?.resolve(null);
+ this.resolver = null;
+ };
+
+ private handleError(e?: DOMException) {
+ console.error(e?.message);
+ this.dispose();
+ }
+
+ public async init(track: InputVideoTrack) {
+ this.config = await track.getDecoderConfig();
+ assert(this.config, 'Failed to get decoder config from track');
+ const support = await VideoDecoder.isConfigSupported(this.config);
+ assert(support.supported, 'Decoder config not supported');
+ }
+
+ private ensureDecoder(packet: EncodedPacket) {
+ if (this.decoder || packet.type === 'delta') return;
+
+ assert(this.config, 'Decoder config not available');
+
+ this.decoder = new VideoDecoder({
+ error: this.handleError.bind(this),
+ output: this.handleOutput.bind(this),
+ });
+
+ this.decoder.addEventListener('dequeue', this.handleDequeue.bind(this));
+ this.decoder.configure(this.config);
+ }
+
+ public async decode(packet: EncodedPacket) {
+ this.ensureDecoder(packet);
+
+ if (this.decoder?.state !== 'configured') {
+ this.lastSubmitted = null;
+ this.resolver?.resolve(null);
+ this.resolver = null;
+ this.inFlight.clear();
+ return;
+ }
+
+ if (this.decoder.decodeQueueSize > 2) {
+ this.resolver = Promise.withResolvers();
+ }
+
+ this.decoder.decode(packet.toEncodedVideoChunk());
+ this.lastSubmitted = packet;
+ this.inFlight.add(packet.microsecondTimestamp);
+
+ await this.resolver?.promise;
+ }
+
+ public dispose() {
+ try {
+ this.decoder?.close();
+ } catch { /* ignore */ }
+ this.resolver?.resolve(null);
+ this.decoder = null;
+ this.lastSubmitted = null;
+ this.resolver = null;
+ this.inFlight.clear();
+ }
+}
+
+/**
+ * Dedicated, full-resolution video decoder for export.
+ */
+export class VideoExporter {
+ public errored = false;
+ public asset: VideoAsset;
+ public initialized: Promise<void>;
+
+ private input: Input | null = null;
+ private canvasSink: CanvasSink | null = null;
+ private iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null = null;
+ private currentCanvas: WrappedCanvas | null = null;
+ private firstTimestamp: number = 0;
+
+ public constructor(asset: VideoAsset) {
+ this.asset = asset;
+ this.initialized = this.initialize();
+ }
+
+ private async initialize() {
+ try {
+ const blob = await getAssetFile(this.asset);
+ this.input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
+ const track = await this.input.getPrimaryVideoTrack();
+ assert(track, 'Video track not found');
+ // See VideoBuffer.initialize: clamp so an edit-list head trim (negative first
+ // timestamp) doesn't offset every exported frame relative to the audio.
+ this.firstTimestamp = Math.max(0, await track.getFirstTimestamp() ?? 0);
+ this.canvasSink = new CanvasSink(track, { poolSize: 2 });
+ } catch (e) {
+ console.error('Error initializing video exporter', e);
+ this.errored = true;
+ }
+ }
+
+ public async seekTo(frame: number, frameRate: number): Promise<void> {
+ await this.initialized;
+
+ if (this.errored || !this.canvasSink) return;
+
+ const targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);
+ const lastFrame = this.currentCanvas
+ ? this.secondsToFrames(this.currentCanvas.timestamp)
+ : -1;
+
+ if (targetFrame === lastFrame) return;
+
+ if (!this.iterator || targetFrame < lastFrame) {
+ this.iterator = this.canvasSink?.canvases(this.framesToSeconds(targetFrame)) ?? null;
+ }
+ if (!this.iterator) return;
+
+ // Walk forward while the lookahead frame still starts at/before target.
+ while (true) {
+ const { value, done } = await this.iterator.next();
+ if (done || !value) break;
+
+ this.currentCanvas = value;
+
+ if (this.secondsToFrames(value.timestamp) >= targetFrame) {
+ break;
+ }
+ }
+ }
+
+ private framesToSeconds(frames: number) {
+ return Math.max(this.firstTimestamp, (frames / this.asset.frameRate) + this.firstTimestamp);
+ }
+
+ private secondsToFrames(seconds: number) {
+ return Math.max(0, Math.round((seconds - this.firstTimestamp) * this.asset.frameRate));
+ }
+
+ public toBitmap(): HTMLCanvasElement | OffscreenCanvas | null {
+ if (!this.currentCanvas) {
+ return null;
+ }
+
+ return this.currentCanvas.canvas;
+ }
+
+ public idle(): void { }
+
+ public dispose(): void {
+ this.iterator?.return();
+ this.iterator = null;
+ this.input?.dispose();
+ }
+}
+
+export type VideoDecoderInstance = VideoBuffer | VideoExporter;
+
+const videoTrackCache = new Map<string, Promise<InputVideoTrack | null>>();
+
+export function clearVideoTrackCache() {
+ videoTrackCache.clear();
+}
+
+export function getVideoTrack(source: VideoAsset) {
+ let promise = videoTrackCache.get(source.id);
+ if (promise) {
+ return promise;
+ }
+
+ promise = (async () => {
+ try {
+ const blob = await getAssetFile(source);
+ const input = new Input({
+ formats: ALL_FORMATS,
+ source: new BlobSource(blob)
+ });
+ return await input.getPrimaryVideoTrack();
+ } catch {
+ videoTrackCache.delete(source.id);
+ return null;
+ }
+ })();
+
+ videoTrackCache.set(source.id, promise);
+ return promise;
+}
+
+export function resolveVideoDecoder(world: World, entity: Entity): VideoDecoderInstance | null {
+ const assetId = entity.get(AssetId)?.value;
+ if (!assetId) return null;
+
+ const existing = entity.get(VideoDecoderHandle);
+ if (existing && existing.asset.id === assetId) {
+ return existing;
+ }
+
+ // Asset changed — dispose old decoder and create a new one.
+ existing?.dispose();
+
+ const asset = world.get(Assets)?.get(assetId);
+ if (!asset || asset.type !== 'VIDEO') return null;
+
+ const decoder = world.get(Mode)?.value === 'realtime'
+ ? new VideoBuffer(asset)
+ : new VideoExporter(asset);
+ entity.add(VideoDecoderHandle);
+ entity.set(VideoDecoderHandle, decoder);
+ return decoder;
+}
diff --git a/packages/runtime/src/systems/gradients.ts b/packages/runtime/src/systems/gradients.ts
new file mode 100644
index 00000000..9a2a0e5f
--- /dev/null
+++ b/packages/runtime/src/systems/gradients.ts
@@ -0,0 +1,130 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Canvas gradient construction from paint sub-entities (was part of
+// systems/render.ts; split out because text rendering needs it before the
+// render system moves in).
+
+import { Not } from 'koota';
+
+import { store } from '../world/store';
+import { ChildOf, Deleted, ColorStop, Position, Computed } from '../traits';
+import { colorToHex } from '../utils/color';
+
+import type { Entity, World } from 'koota';
+
+type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+
+function addStopsTo(world: World, fill: Entity, gradient: CanvasGradient): void {
+ const computed = store(world, Computed);
+ const stops = [...world.query(ColorStop, ChildOf(fill), Not(Deleted))]
+ .map(stop => {
+ const raw = computed.stopOffset[stop.id()]!;
+ return {
+ offset: raw <= 1 ? Math.max(0, raw) : raw % 1,
+ color: computed.stopColor[stop.id()]!,
+ opacity: computed.stopOpacity[stop.id()]!,
+ };
+ })
+ .sort((a, b) => a.offset - b.offset);
+
+ for (const { offset, color, opacity } of stops) {
+ gradient.addColorStop(offset, toGradientColor(color, opacity));
+ }
+}
+
+function toGradientColor(color: number, opacity: number): string {
+ if (opacity >= 1) return colorToHex(color);
+
+ const r = (color >> 16) & 0xFF;
+ const g = (color >> 8) & 0xFF;
+ const b = color & 0xFF;
+ return `rgba(${r},${g},${b},${Math.max(0, Math.min(1, opacity))})`;
+}
+
+/** Create a canvas linear gradient from a gradient paint sub-entity. */
+export function createLinearGradient(
+ world: World,
+ fill: Entity,
+ ctx: Ctx2D,
+ w: number,
+ h: number,
+): CanvasGradient {
+ const computed = store(world, Computed);
+ const fid = fill.id();
+
+ // (px, py) is the gradient center in normalized shape-local space. Fills
+ // without a Position trait (the default for stock gradient fills) get
+ // (0.5, 0.5) so the gradient spans the shape. With Position present, the
+ // motion-system-sampled Computed values are used (keyframeable).
+ const hasPos = fill.has(Position);
+ const px = hasPos ? computed.positionX[fid]! : 0.5;
+ const py = hasPos ? computed.positionY[fid]! : 0.5;
+ const sx = computed.scaleX[fid]!;
+ const sy = computed.scaleY[fid]!;
+ const angle = (computed.rotation[fid]! * Math.PI) / 180;
+ const cos = Math.cos(angle);
+ const sin = Math.sin(angle);
+
+ // Transform normalized start (0, 0.5) and end (1, 0.5) points
+ const startLocalX = (0 - 0.5) * sx;
+ const startLocalY = (0.5 - 0.5) * sy;
+ const x0 = (px + startLocalX * cos - startLocalY * sin) * w;
+ const y0 = (py + startLocalX * sin + startLocalY * cos) * h;
+
+ const endLocalX = (1 - 0.5) * sx;
+ const endLocalY = (0.5 - 0.5) * sy;
+ const x1 = (px + endLocalX * cos - endLocalY * sin) * w;
+ const y1 = (py + endLocalX * sin + endLocalY * cos) * h;
+
+ const gradient = ctx.createLinearGradient(x0, y0, x1, y1);
+
+ addStopsTo(world, fill, gradient);
+ return gradient;
+}
+
+/** Create a canvas radial gradient from a gradient paint sub-entity. */
+export function createRadialGradient(
+ world: World,
+ fill: Entity,
+ ctx: Ctx2D,
+ w: number,
+ h: number,
+): CanvasGradient {
+ const computed = store(world, Computed);
+ const fid = fill.id();
+
+ const hasPos = fill.has(Position);
+ const px = hasPos ? computed.positionX[fid]! : 0.5;
+ const py = hasPos ? computed.positionY[fid]! : 0.5;
+ const sx = computed.scaleX[fid]!;
+ const sy = computed.scaleY[fid]!;
+ const angle = (computed.rotation[fid]! * Math.PI) / 180;
+ const cos = Math.cos(angle);
+ const sin = Math.sin(angle);
+
+ // Center point (0.5, 0.5) in normalized space
+ const cx = px * w;
+ const cy = py * h;
+
+ // Edge points to determine radius
+ const edgeXLocalX = (1 - 0.5) * sx;
+ const edgeXLocalY = (0.5 - 0.5) * sy;
+ const edgeXAbsX = (px + edgeXLocalX * cos - edgeXLocalY * sin) * w;
+ const edgeXAbsY = (py + edgeXLocalX * sin + edgeXLocalY * cos) * h;
+
+ const edgeYLocalX = (0.5 - 0.5) * sx;
+ const edgeYLocalY = (1 - 0.5) * sy;
+ const edgeYAbsX = (px + edgeYLocalX * cos - edgeYLocalY * sin) * w;
+ const edgeYAbsY = (py + edgeYLocalX * sin + edgeYLocalY * cos) * h;
+
+ const distX = Math.hypot(edgeXAbsX - cx, edgeXAbsY - cy);
+ const distY = Math.hypot(edgeYAbsX - cx, edgeYAbsY - cy);
+ const outerRadius = Math.max(0.0001, distX, distY);
+
+ const gradient = ctx.createRadialGradient(cx, cy, 0, cx, cy, outerRadius);
+
+ addStopsTo(world, fill, gradient);
+ return gradient;
+}
diff --git a/packages/runtime/src/systems/index.ts b/packages/runtime/src/systems/index.ts
index dffd197c..7df32c4a 100644
--- a/packages/runtime/src/systems/index.ts
+++ b/packages/runtime/src/systems/index.ts
@@ -1,3 +1,4 @@
// Deterministic systems. Browser-only systems (input, keyboard, camera, hud, dom) stay in apps/web.
export * from './transform';
export * from './motion';
+export * from './gradients';
diff --git a/packages/runtime/src/traits/media.ts b/packages/runtime/src/traits/media.ts
index 6ad6679d..048e4fa9 100644
--- a/packages/runtime/src/traits/media.ts
+++ b/packages/runtime/src/traits/media.ts
@@ -4,6 +4,18 @@
import { trait } from 'koota';
+// Type-only imports: erased at compile time, so the traits ↔ media module
+// cycle never materializes at runtime.
+import type { HtmlHost } from '../media/html';
+import type { SurfaceHost } from '../media/surface';
+import type { ShaderHost } from '../media/shader';
+import type { ImageDecoder } from '../media/image';
+import type { VideoDecoderInstance } from '../media/video';
+import type { SequenceDecoder } from '../media/sequence';
+import type { AudioDecoder } from '../media/audio';
+import type { CaptionDecoder } from '../media/caption/types';
+import type { AudioBus } from '../media/audio-bus';
+
// Renderer-runtime state on mounted entities (never serialized).
export type MountData = {
props: Record<string, unknown>;
@@ -12,26 +24,25 @@ export type MountData = {
// Handle traits store live runtime instances (hosts, decoders, buses) and are
// never serialized. They use AoS storage: entity.get(X) returns the instance
-// directly. Handle types are opaque until engine/decoders and services move
-// into src/media; tighten them to the concrete classes then.
-type Handle = object;
+// directly. Suffixed Handle so the trait and the class it stores (HtmlHost,
+// AudioDecoder, ...) can share one export surface.
-export const HtmlHost = trait(() => null as Handle | null);
+export const HtmlHostHandle = trait(() => null as HtmlHost | null);
-export const SurfaceHost = trait(() => null as Handle | null);
+export const SurfaceHostHandle = trait(() => null as SurfaceHost | null);
-export const ShaderHost = trait(() => null as Handle | null);
+export const ShaderHostHandle = trait(() => null as ShaderHost | null);
-export const ImageDecoder = trait(() => null as Handle | null);
+export const ImageDecoderHandle = trait(() => null as ImageDecoder | null);
-export const VideoDecoder = trait(() => null as Handle | null);
+export const VideoDecoderHandle = trait(() => null as VideoDecoderInstance | null);
-export const SequenceDecoder = trait(() => null as Handle | null);
+export const SequenceDecoderHandle = trait(() => null as SequenceDecoder | null);
-export const AudioDecoder = trait(() => null as Handle | null);
+export const AudioDecoderHandle = trait(() => null as AudioDecoder | null);
-export const CaptionDecoder = trait(() => null as Handle | null);
+export const CaptionDecoderHandle = trait(() => null as CaptionDecoder | null);
-export const AudioBus = trait(() => null as Handle | null);
+export const AudioBusHandle = trait(() => null as AudioBus | null);
export const Data = trait(() => null as MountData | null);
diff --git a/packages/runtime/src/traits/text.ts b/packages/runtime/src/traits/text.ts
index d8400819..d8092397 100644
--- a/packages/runtime/src/traits/text.ts
+++ b/packages/runtime/src/traits/text.ts
@@ -13,17 +13,20 @@ export const Chars = trait({ value: '' });
// Text style properties, shared between a text node and its TextRange
// sub-entities. On a text node this is the default style; on a range
-// sub-entity it overrides.
+// sub-entity it overrides. All fields default to undefined: a range override
+// carries ONLY its explicitly set fields (a sparse store slot means "inherit
+// from the node"), and the node-level fallbacks (16px, Inter, 400, ...) live
+// in the text renderer's accessors.
export const TextStyle = trait({
- leading: 1.2,
- fontSize: 16,
- fontFamily: '',
- fontWeight: '400',
- fontStyle: FontStyle.NORMAL as FontStyle,
- textAlign: TextAlign.LEFT as TextAlign,
- textBaseline: TextBaseline.TOP as TextBaseline,
- textCase: TextCase.ORIGINAL as TextCase,
- letterSpacing: 0, // extra spacing between characters (px)
+ leading: undefined as number | undefined,
+ fontSize: undefined as number | undefined,
+ fontFamily: undefined as string | undefined,
+ fontWeight: undefined as string | undefined,
+ fontStyle: undefined as FontStyle | undefined,
+ textAlign: undefined as TextAlign | undefined,
+ textBaseline: undefined as TextBaseline | undefined,
+ textCase: undefined as TextCase | undefined,
+ letterSpacing: undefined as number | undefined, // extra spacing between characters (px)
});
// Character range a TextStyle override applies to; end null = to the end.
diff --git a/packages/runtime/src/utils/async.ts b/packages/runtime/src/utils/async.ts
new file mode 100644
index 00000000..a369856e
--- /dev/null
+++ b/packages/runtime/src/utils/async.ts
@@ -0,0 +1,35 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Attempt to execute a function and return the result or the error it threw.
+ */
+export function attempt<T>(fn: () => T): { result: T } | { error: Error } {
+ try {
+ return { result: fn() };
+ } catch (error) {
+ if (error instanceof Error) {
+ return { error };
+ }
+ return { error: new Error('Unknown error') };
+ }
+}
+
+export class AsyncMutex {
+ currentPromise = Promise.resolve();
+
+ async acquire() {
+ let resolver: () => void;
+ const nextPromise = new Promise<void>(resolve => {
+ resolver = resolve;
+ });
+
+ const currentPromiseAlias = this.currentPromise;
+ this.currentPromise = nextPromise;
+
+ await currentPromiseAlias;
+
+ return resolver!;
+ }
+}
diff --git a/packages/runtime/src/utils/color.ts b/packages/runtime/src/utils/color.ts
new file mode 100644
index 00000000..08b145c6
--- /dev/null
+++ b/packages/runtime/src/utils/color.ts
@@ -0,0 +1,120 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Packed 0xRRGGBB color helpers (subset of the app's utils/color.ts that the
+// renderer and shader uniforms need; UI-facing conversions stay app-side).
+
+import { colord } from 'colord';
+
+import { clamp } from '../math/common';
+
+const NAMED_COLORS: Record<string, string> = {
+ red: 'FF0000',
+ green: '008000',
+ blue: '0000FF',
+ white: 'FFFFFF',
+ black: '000000',
+ yellow: 'FFFF00',
+ cyan: '00FFFF',
+ magenta: 'FF00FF',
+ orange: 'FFA500',
+ purple: '800080',
+ pink: 'FFC0CB',
+ gray: '808080',
+ grey: '808080',
+ brown: 'A52A2A',
+ lime: '00FF00',
+ navy: '000080',
+ teal: '008080',
+ maroon: '800000',
+ olive: '808000',
+ aqua: '00FFFF',
+ coral: 'FF7F50',
+ salmon: 'FA8072',
+ gold: 'FFD700',
+ silver: 'C0C0C0',
+ indigo: '4B0082',
+ violet: 'EE82EE',
+ crimson: 'DC143C',
+ tomato: 'FF6347',
+ khaki: 'F0E68C',
+ plum: 'DDA0DD',
+ orchid: 'DA70D6',
+ tan: 'D2B48C',
+ beige: 'F5F5DC',
+ ivory: 'FFFFF0',
+ lavender: 'E6E6FA',
+ turquoise: '40E0D0',
+ chartreuse: '7FFF00',
+ sienna: 'A0522D',
+ peru: 'CD853F',
+ firebrick: 'B22222',
+ skyblue: '87CEEB',
+ hotpink: 'FF69B4',
+ mint: '3EB489',
+};
+
+/**
+ * Parses any colord-compatible input (hex with or without `#`, short hex,
+ * `rgb()`, `hsl()`, named colors, etc.) into a numeric hex color (0xRRGGBB).
+ * Arbitrary strings fall back to Figma-style hex normalization: strip non-hex
+ * characters, then repeat/pad to six digits (empty input returns null).
+ */
+export function parseColor(input: string): number | null {
+ const trimmed = input.trim();
+ if (trimmed.length === 0) return null;
+
+ const named = NAMED_COLORS[trimmed.toLowerCase()];
+ if (named) {
+ return parseInt(named, 16);
+ }
+
+ const c = colord(trimmed);
+ if (c.isValid()) {
+ const { r, g, b } = c.toRgb();
+ return rgbToColor(r, g, b);
+ }
+
+ const hex = trimmed.replace(/[^0-9A-Fa-f]/g, '');
+ if (hex.length === 0) return null;
+
+ let normalized: string;
+ if (hex.length === 1) normalized = hex.repeat(6);
+ else if (hex.length === 2) normalized = hex.repeat(3);
+ else if (hex.length === 3) normalized = hex.split('').map((ch) => ch + ch).join('');
+ else if (hex.length === 4) normalized = hex + '00';
+ else if (hex.length === 5) normalized = hex + '0';
+ else if (hex.length === 6) normalized = hex;
+ else normalized = hex.slice(0, 6);
+
+ return parseInt(normalized.toUpperCase(), 16);
+}
+
+/**
+ * Converts a numeric hex color (0xRRGGBB) to a CSS hex string (#RRGGBB).
+ */
+export function colorToHex(color: number): string {
+ return '#' + ((1 << 24) | (color & 0xFFFFFF)).toString(16).slice(1).toUpperCase();
+}
+
+/**
+ * Packs RGB channel values (0-255) into a numeric hex color (0xRRGGBB).
+ */
+export function rgbToColor(r: number, g: number, b: number): number {
+ const rc = clamp(Math.round(r), 0, 255);
+ const gc = clamp(Math.round(g), 0, 255);
+ const bc = clamp(Math.round(b), 0, 255);
+ return (rc << 16) | (gc << 8) | bc;
+}
+
+/**
+ * Unpacks a numeric hex color (0xRRGGBB) into RGB channel values (0-255).
+ */
+export function colorToRgb(color: number): { r: number; g: number; b: number } {
+ return {
+ r: (color >> 16) & 0xFF,
+ g: (color >> 8) & 0xFF,
+ b: color & 0xFF,
+ };
+}
diff --git a/packages/runtime/src/utils/index.ts b/packages/runtime/src/utils/index.ts
index a458d2f2..6487bfec 100644
--- a/packages/runtime/src/utils/index.ts
+++ b/packages/runtime/src/utils/index.ts
@@ -1,5 +1,7 @@
// Pure helpers only: transition, captions follow. DOM-touching utils (cursor, interaction) stay in apps/web.
export * from './assert';
+export * from './async';
+export * from './color';
export * from './text';
export * from './live-mounts';
export * from './sort';
diff --git a/packages/runtime/src/utils/text.ts b/packages/runtime/src/utils/text.ts
index b53cd454..46473328 100644
--- a/packages/runtime/src/utils/text.ts
+++ b/packages/runtime/src/utils/text.ts
@@ -2,8 +2,25 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Text layout data model. The layout, measurement, and canvas render helpers
-// join this file when engine/utils/text.ts moves into the runtime.
+// Text layout, measurement, and canvas rendering (was engine/utils/text.ts).
+
+import { store } from '../world/store';
+import {
+ COMPOSITE_OPERATIONS, PaintType, FontStyle, StrokeCap, StrokeJoin,
+ TextAlign, TextBaseline, TextCase,
+} from '../constants';
+import {
+ Size, Hidden, Paint, Color, Blur, Offset, Appearance, StrokeStyle,
+ Chars, TextStyle, TextRange, TextCache, Cache, Computed, Camera,
+ RenderSurface,
+} from '../traits';
+import { clamp } from '../math/common';
+import { colorToHex } from './color';
+import { createLinearGradient, createRadialGradient } from '../systems/gradients';
+
+import type { Entity, World } from 'koota';
+
+type Ctx = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
export type TokenOptions = {
/**
@@ -19,9 +36,9 @@ export type TokenOptions = {
*/
metrics: TextMetrics;
/**
- * Defines the style of the token
+ * Defines the style of the token (TextRange sub-entities)
*/
- ranges: number[];
+ ranges: Entity[];
}
export type Line = {
@@ -31,10 +48,27 @@ export type Line = {
height: number;
};
+type RenderSplit = {
+ words: string[];
+ ranges: Entity[];
+};
+
+/** Global offscreen canvas used solely for text measurement. */
+let measureCanvas: OffscreenCanvas | null = null;
+let measureCtx: OffscreenCanvasRenderingContext2D | null = null;
+
+function getMeasureCtx(): OffscreenCanvasRenderingContext2D {
+ if (!measureCtx) {
+ measureCanvas = new OffscreenCanvas(1, 1);
+ measureCtx = measureCanvas.getContext('2d', { willReadFrequently: true, alpha: true })!;
+ }
+ return measureCtx;
+}
+
export class Token {
public offset: number;
public metrics: TextMetrics;
- public ranges: number[];
+ public ranges: Entity[];
public chars: string;
public width: number;
public height: number;
@@ -93,3 +127,559 @@ export class Token {
this.line = { ...line };
}
}
+
+function applyFont(ctx: Ctx, world: World, entity: Entity, ranges: Entity[]) {
+ const size = getFontSize(world, entity, ranges);
+ const family = getFontFamily(world, entity, ranges);
+ const weight = getFontWeight(world, entity, ranges);
+ const style = getFontStyle(world, entity, ranges);
+ const baseline = getTextBaseline(world, entity, ranges);
+ const spacing = getLetterSpacing(world, entity, ranges);
+
+ const mappedStyle = FontStyle[style]!.toLowerCase();
+ const mappedBaseline = TextBaseline[baseline]!.toLowerCase() as CanvasTextBaseline;
+
+ ctx.font = `${mappedStyle} ${weight.toLowerCase()} ${size}px ${family}`.trim();
+ ctx.textBaseline = mappedBaseline;
+ ctx.letterSpacing = `${spacing}px`;
+}
+
+function applyStroke(ctx: Ctx, world: World, entity: Entity) {
+ const stroke = store(world, StrokeStyle);
+ const eid = entity.id();
+
+ ctx.lineWidth = stroke.width[eid] ?? 1;
+ ctx.lineJoin = StrokeJoin[stroke.join[eid] ?? 0]!.toLocaleLowerCase() as CanvasLineJoin;
+ ctx.lineCap = StrokeCap[stroke.cap[eid] ?? 0]!.toLocaleLowerCase() as CanvasLineCap;
+ ctx.miterLimit = stroke.miterLimit[eid] ?? 3;
+}
+
+function shapeTokens(world: World, entity: Entity): void {
+ const computed = store(world, Computed);
+ const textStyle = store(world, TextStyle);
+ const eid = entity.id();
+ const lines = store(world, TextCache).tokens[eid];
+ const leading = textStyle.leading[eid] ?? 1;
+ const textAlign = textStyle.textAlign[eid] ?? TextAlign.LEFT;
+ const textBaseline = textStyle.textBaseline[eid] ?? TextBaseline.TOP;
+ if (!lines) return;
+
+ // Measure each line's width by summing word widths and spaces
+ const lineWidths: number[] = lines.map(line => line.reduce((acc, word) => acc + word.width, 0));
+ const lineHeights: number[] = lines.map(line => Math.max(...line.map(word => word.height)));
+
+ // Find the maximum line width
+ const maxLineWidth = Math.max(...lineWidths);
+
+ // Calculate total height based on leading
+ const totalHeight = lineHeights.reduce((acc, height, i) => acc + height * (i < lineHeights.length - 1 ? leading : 1), 0);
+
+ // When Size is assigned, use Computed for positioning; otherwise set it from text dimensions
+ if (!entity.has(Size) || !computed.width[eid] || !computed.height[eid]) {
+ computed.width[eid] = Math.ceil(maxLineWidth);
+ computed.height[eid] = Math.ceil(totalHeight);
+ }
+
+ const containerWidth = computed.width[eid]!;
+ const containerHeight = computed.height[eid]!;
+
+ // Calculate initial vertical offset based on text baseline and container height
+ let offsetY = 0;
+ if (textBaseline === TextBaseline.MIDDLE) {
+ offsetY = (containerHeight - totalHeight) / 2;
+ } else if (textBaseline === TextBaseline.BOTTOM) {
+ offsetY = containerHeight - totalHeight;
+ }
+
+ // Calculate coordinates for each word
+ const line = {
+ offsetX: 0,
+ offsetY,
+ baseline: 0,
+ height: 0,
+ };
+
+ for (let l = 0; l < lines.length; l++) {
+ line.height = lineHeights[l]!;
+
+ if (textAlign === TextAlign.LEFT) {
+ line.offsetX = 0;
+ } else if (textAlign === TextAlign.CENTER) {
+ line.offsetX = (containerWidth - lineWidths[l]!) / 2;
+ } else if (textAlign === TextAlign.RIGHT) {
+ line.offsetX = containerWidth - lineWidths[l]!;
+ }
+
+ if (textBaseline === TextBaseline.TOP) {
+ line.baseline = 0;
+ } else if (textBaseline === TextBaseline.MIDDLE) {
+ line.baseline = lineHeights[l]! / 2;
+ } else if (textBaseline === TextBaseline.BOTTOM) {
+ line.baseline = lineHeights[l]!;
+ } else if (textBaseline === TextBaseline.ALPHABETIC) {
+ const maxAscent = Math.max(...lines[l]!.map(word => word.metrics.fontBoundingBoxAscent));
+ line.baseline = maxAscent || lineHeights[l]! * 0.75; // Fallback if no words in line
+ }
+
+ for (const word of lines[l]!) {
+ word.setLine(line);
+ }
+
+ line.offsetY += lineHeights[l]! * leading;
+ }
+}
+
+function tokenizeText(world: World, entity: Entity) {
+ const ctx = getMeasureCtx();
+
+ // Split text into segments based on style overrides first
+ const lines: Token[][] = [[]];
+
+ const maxWidth = store(world, Size).width[entity.id()] ?? Number.POSITIVE_INFINITY;
+
+ let offset = 0;
+ for (const { words, ranges } of createRenderSplits(world, entity)) {
+ applyFont(ctx, world, entity, ranges);
+
+ for (const word of words) {
+ const splits = word.split('\n');
+
+ for (let i = 0; i < splits.length; i++) {
+ const caseValue = getTextCase(world, entity, ranges);
+ const chars = transformText(splits[i]!, caseValue);
+
+ const metrics = ctx.measureText(chars);
+
+ // check if word is too wide for the current line but not the first word
+ if (offset + metrics.width > maxWidth && offset > 0) {
+ lines.push([]);
+ offset = 0;
+ }
+
+ // add word to current line
+ lines[lines.length - 1]!.push(new Token({ chars, ranges, metrics, offset }));
+ offset += metrics.width;
+
+ // add new line if not the last line
+ if (i < splits.length - 1) {
+ lines.push([]);
+ offset = 0;
+ }
+ }
+ }
+ }
+
+ if (!entity.has(TextCache)) entity.add(TextCache);
+ store(world, TextCache).tokens[entity.id()] = lines;
+}
+
+/** Renders text tokens directly to the given canvas context. */
+function renderTokens(ctx: Ctx, world: World, entity: Entity): void {
+ const eid = entity.id();
+ const lines = store(world, TextCache).tokens[eid];
+ if (!lines) return;
+ const words = lines.flat();
+
+ const computed = store(world, Computed);
+ const offsetStore = store(world, Offset);
+ const blurStore = store(world, Blur);
+ const colorStore = store(world, Color);
+ const appearance = store(world, Appearance);
+ const paintStore = store(world, Paint);
+
+ const savedAlpha = ctx.globalAlpha;
+
+ // Draw all text shadows
+ {
+ ctx.save();
+ ctx.textAlign = 'start';
+ ctx.textBaseline = 'top';
+
+ // ctx.shadowBlur/OffsetX/OffsetY are in device-pixel space and are not
+ // affected by the current transform, so scale them up to match the
+ // content transform (camera * resolution).
+ const camera = world.get(Camera);
+ const resolution = world.get(RenderSurface)?.resolution ?? 1;
+ const shadowScale = (camera?.a ?? 1) * resolution;
+
+ for (const word of words) {
+ applyFont(ctx, world, entity, word.ranges);
+
+ const strokes = getStrokes(world, entity, word.ranges);
+ const shadows = getShadows(world, entity, word.ranges);
+
+ if (strokes.length) {
+ applyStroke(ctx, world, entity);
+ }
+
+ // Draw shadows first (if any)
+ for (const shadow of shadows) {
+ if (shadow.has(Hidden)) continue;
+ const sid = shadow.id();
+
+ ctx.shadowOffsetX = (offsetStore.x[sid] ?? 0) * shadowScale;
+ ctx.shadowOffsetY = (offsetStore.y[sid] ?? 0) * shadowScale;
+ ctx.shadowBlur = (blurStore.value[sid] ?? 0) * shadowScale;
+ ctx.shadowColor = colorToHex(colorStore.value[sid] ?? 0x000000);
+ ctx.fillStyle = colorToHex(colorStore.value[sid] ?? 0x000000);
+ ctx.globalAlpha = savedAlpha * (appearance.opacity[sid] ?? 1);
+
+ if (strokes.length) {
+ ctx.strokeText(word.chars, word.x, word.y);
+ } else {
+ ctx.fillText(word.chars, word.x, word.y);
+ }
+ }
+
+ // Reset shadow properties if any shadows are applied
+ if (shadows.length) {
+ ctx.globalAlpha = savedAlpha;
+ ctx.shadowColor = 'transparent';
+ }
+ }
+ ctx.restore();
+ }
+
+ // Draw all text strokes
+ {
+ ctx.save();
+ ctx.textAlign = 'start';
+ ctx.textBaseline = 'top';
+
+ for (const word of words) {
+ const strokes = getStrokes(world, entity, word.ranges);
+ if (!strokes.length) continue;
+
+ applyFont(ctx, world, entity, word.ranges);
+ applyStroke(ctx, world, entity);
+
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+
+ // Draw strokes (if any)
+ for (const stroke of strokes) {
+ if (stroke.has(Hidden)) continue;
+ const sid = stroke.id();
+
+ const savedCO = ctx.globalCompositeOperation;
+ const blendMode = appearance.blendMode[sid] ?? 0;
+ if (blendMode !== 0) {
+ ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[blendMode]!;
+ }
+ ctx.globalAlpha = savedAlpha * (appearance.opacity[sid] ?? 1);
+
+ const paintType = paintStore.value[sid];
+ if (paintType === PaintType.LINEAR_GRADIENT) {
+ ctx.strokeStyle = createLinearGradient(world, stroke, ctx, w, h);
+ } else if (paintType === PaintType.RADIAL_GRADIENT) {
+ ctx.strokeStyle = createRadialGradient(world, stroke, ctx, w, h);
+ } else {
+ ctx.strokeStyle = colorToHex(colorStore.value[sid] ?? 0x000000);
+ }
+ ctx.strokeText(word.chars, word.x, word.y);
+ ctx.globalCompositeOperation = savedCO;
+ }
+ }
+ ctx.restore();
+ }
+
+ // Draw all text fills
+ {
+ ctx.save();
+ ctx.textAlign = 'start';
+ ctx.textBaseline = 'top';
+
+ for (const word of words) {
+ const fills = getFills(world, entity, word.ranges);
+ if (!fills.length) continue;
+
+ applyFont(ctx, world, entity, word.ranges);
+
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+
+ for (const fill of fills) {
+ if (fill.has(Hidden)) continue;
+ const fid = fill.id();
+
+ const savedCO = ctx.globalCompositeOperation;
+ const blendMode = appearance.blendMode[fid] ?? 0;
+ if (blendMode !== 0) {
+ ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[blendMode]!;
+ }
+ ctx.globalAlpha = savedAlpha * (appearance.opacity[fid] ?? 1);
+
+ const paintType = paintStore.value[fid];
+ if (paintType === PaintType.LINEAR_GRADIENT) {
+ ctx.fillStyle = createLinearGradient(world, fill, ctx, w, h);
+ } else if (paintType === PaintType.RADIAL_GRADIENT) {
+ ctx.fillStyle = createRadialGradient(world, fill, ctx, w, h);
+ } else {
+ ctx.fillStyle = colorToHex(colorStore.value[fid] ?? 0x000000);
+ }
+ ctx.fillText(word.chars, word.x, word.y);
+ ctx.globalCompositeOperation = savedCO;
+ }
+ }
+ ctx.restore();
+ }
+}
+
+/** Render text tokens directly to the world's render surface. */
+export function renderText(world: World, entity: Entity) {
+ const ctx = world.get(RenderSurface)?.ctx;
+ if (!ctx) return;
+ tokenizeText(world, entity);
+ shapeTokens(world, entity);
+ renderTokens(ctx, world, entity);
+}
+
+/**
+ * Splits text into segments based on overlapping text range entities.
+ * Handles overlapping ranges by merging them.
+ * @returns Array of RenderSplit objects with words and contributing ranges
+ */
+function createRenderSplits(world: World, entity: Entity): RenderSplit[] {
+ const eid = entity.id();
+ const chars = store(world, Computed).chars[eid] ?? store(world, Chars).value[eid] ?? '';
+
+ const textRanges = store(world, Cache).textRanges[eid] ?? [];
+
+ // Fast path: no styles
+ if (textRanges.length === 0) {
+ return [{
+ words: tokenize(chars),
+ ranges: [],
+ }];
+ }
+
+ // Fast path: empty text
+ if (chars.length === 0) {
+ return [{
+ words: [],
+ ranges: [],
+ }];
+ }
+
+ const rangeStore = store(world, TextRange);
+
+ // Collect all unique boundary points
+ const boundaries = new Set<number>();
+ boundaries.add(0);
+ boundaries.add(chars.length);
+
+ for (const range of textRanges) {
+ let start = rangeStore.start[range.id()] ?? 0;
+ let end = rangeStore.end[range.id()] ?? chars.length;
+
+ // Clamp boundaries to valid text range
+ start = clamp(start, 0, chars.length);
+ end = clamp(end, 0, chars.length);
+
+ boundaries.add(start);
+ boundaries.add(end);
+ }
+
+ // Sort boundaries in ascending order
+ const sortedBoundaries = Array.from(boundaries).sort((a, b) => a - b);
+
+ // Create segments between each pair of boundaries
+ const segments: RenderSplit[] = [];
+
+ for (let i = 0; i < sortedBoundaries.length - 1; i++) {
+ const segStart = sortedBoundaries[i]!;
+ const segEnd = sortedBoundaries[i + 1]!;
+
+ // Skip empty segments
+ if (segStart >= segEnd) continue;
+
+ // Find all active styles for this segment
+ const ranges: Entity[] = [];
+
+ for (const range of textRanges) {
+ let start = rangeStore.start[range.id()] ?? 0;
+ let end = rangeStore.end[range.id()] ?? chars.length;
+
+ // Clamp boundaries to valid text range
+ start = clamp(start, 0, chars.length);
+ end = clamp(end, 0, chars.length);
+
+ // A style is active if segment falls within its range [start, end)
+ // Using < for end to treat ranges as [start, end)
+ if (start <= segStart && end > segStart) {
+ ranges.push(range);
+ }
+ }
+
+ segments.push({
+ words: tokenize(chars.slice(segStart, segEnd)),
+ ranges: ranges,
+ });
+ }
+
+ return segments;
+}
+
+function tokenize(input: string): string[] {
+ // Fast path for inputs without spaces
+ if (input.indexOf(' ') === -1) {
+ return [input];
+ }
+
+ // Use regex to match:
+ // 1. Any characters up to and including a space
+ // 2. OR any remaining characters to the end
+ return input.match(/[^]*? |[^]+$/g) || [input];
+}
+
+export function transformText(text: string, textCase?: number): string {
+ if (textCase == TextCase.LOWER) {
+ return text.toLocaleLowerCase();
+ }
+
+ if (textCase == TextCase.UPPER) {
+ return text.toUpperCase();
+ }
+
+ return text;
+}
+
+function getFills(world: World, entity: Entity, ranges: Entity[]): Entity[] {
+ const cache = store(world, Cache);
+ let value = cache.fills[entity.id()] ?? [];
+
+ for (const range of ranges) {
+ const rangeValue = cache.fills[range.id()];
+ if (rangeValue?.length) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getStrokes(world: World, entity: Entity, ranges: Entity[]): Entity[] {
+ const cache = store(world, Cache);
+ let value = cache.strokes[entity.id()] ?? [];
+
+ for (const range of ranges) {
+ const rangeValue = cache.strokes[range.id()];
+ if (rangeValue?.length) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getShadows(world: World, entity: Entity, ranges: Entity[]): Entity[] {
+ const cache = store(world, Cache);
+ let value = cache.shadows[entity.id()] ?? [];
+
+ for (const range of ranges) {
+ const rangeValue = cache.shadows[range.id()];
+ if (rangeValue?.length) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getTextCase(world: World, entity: Entity, ranges: Entity[]) {
+ const textStyle = store(world, TextStyle);
+ let value = textStyle.textCase[entity.id()] ?? 0;
+
+ for (const range of ranges) {
+ const rangeValue = textStyle.textCase[range.id()];
+ if (rangeValue !== undefined) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getFontSize(world: World, entity: Entity, ranges: Entity[]) {
+ const textStyle = store(world, TextStyle);
+ let value = textStyle.fontSize[entity.id()] ?? 16;
+
+ for (const range of ranges) {
+ const rangeValue = textStyle.fontSize[range.id()];
+ if (rangeValue !== undefined) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getFontFamily(world: World, entity: Entity, ranges: Entity[]) {
+ const textStyle = store(world, TextStyle);
+ let value = textStyle.fontFamily[entity.id()] || 'Inter';
+
+ for (const range of ranges) {
+ const rangeValue = textStyle.fontFamily[range.id()];
+ if (rangeValue !== undefined) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getFontWeight(world: World, entity: Entity, ranges: Entity[]) {
+ const textStyle = store(world, TextStyle);
+ let value = textStyle.fontWeight[entity.id()] ?? '400';
+
+ for (const range of ranges) {
+ const rangeValue = textStyle.fontWeight[range.id()];
+ if (rangeValue !== undefined) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getFontStyle(world: World, entity: Entity, ranges: Entity[]) {
+ const textStyle = store(world, TextStyle);
+ let value = textStyle.fontStyle[entity.id()] ?? FontStyle.NORMAL;
+
+ for (const range of ranges) {
+ const rangeValue = textStyle.fontStyle[range.id()];
+ if (rangeValue !== undefined) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getTextBaseline(world: World, entity: Entity, ranges: Entity[]) {
+ const textStyle = store(world, TextStyle);
+ let value = textStyle.textBaseline[entity.id()] ?? TextBaseline.TOP;
+
+ for (const range of ranges) {
+ const rangeValue = textStyle.textBaseline[range.id()];
+ if (rangeValue !== undefined) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
+
+function getLetterSpacing(world: World, entity: Entity, ranges: Entity[]) {
+ const textStyle = store(world, TextStyle);
+ let value = textStyle.letterSpacing[entity.id()] ?? 0;
+
+ for (const range of ranges) {
+ const rangeValue = textStyle.letterSpacing[range.id()];
+ if (rangeValue !== undefined) {
+ value = rangeValue;
+ }
+ }
+
+ return value;
+}
diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.ts
index 361cfaa2..902df892 100644
--- a/packages/runtime/src/world/observers.ts
+++ b/packages/runtime/src/world/observers.ts
@@ -18,7 +18,7 @@
import { store } from './store';
import {
- ChildOf, Deleted, Sequential, Group, Scene, Audio, Paint, AssetId,
+ ChildOf, Deleted, Culled, Sequential, Group, Scene, Audio, Paint, AssetId,
Delay, PlaybackRate, Trim, Keyframe, ItemIndex,
Position, Offset, Rotation, Scale, UniformScale, Skew, Anchor, Flip,
Appearance, Color, Blur, Volume, Effect, CornerRadius, MixedCornerRadius,
@@ -26,6 +26,10 @@ import {
} from '../traits';
import { getParentEntity } from '../queries/hierarchy';
import { rebuildCaches } from '../actions/cache';
+import {
+ disposeDecoders, disposeHtmlHosts, disposeSurfaceHosts, disposeShaderHosts,
+ disconnectAudioBus,
+} from '../media/dispose';
import {
reactToChildAttached, reactToChildDetached, reactToAssetChange,
reactToPaintChange, recomputeEntityTimeRange, propagateTimeRangeDown,
@@ -49,6 +53,7 @@ export function observeWorld(world: World): () => void {
rebuildCaches(world, child, parent);
propagateSize(world, child);
resolveConstraintOffsets(world, child);
+ disconnectAudioBus(world, child);
reactToChildAttached(world, child);
}));
@@ -56,12 +61,26 @@ export function observeWorld(world: World): () => void {
// still present in the old parent's queries, so it is excluded by hand.
subs.push(world.onRemove(ChildOf('*'), (child, parent) => {
rebuildCaches(world, child, parent, child);
+ disconnectAudioBus(world, child);
}));
subs.push(world.onAdd(Deleted, (entity) => {
// The Not(Deleted) cache queries already skip the tombstone.
rebuildCaches(world, entity, getParentEntity(entity));
reactToChildDetached(world, entity);
+ // A tombstoned subtree keeps its records for undo, but its live media
+ // handles must not linger.
+ disposeDecoders(world, entity);
+ disposeHtmlHosts(world, entity);
+ disposeSurfaceHosts(world, entity);
+ disposeShaderHosts(world, entity);
+ disconnectAudioBus(world, entity);
+ }));
+
+ // Off-screen entities release their decoders (frame caches are the bulk
+ // of media memory); the next resolve recreates them.
+ subs.push(world.onAdd(Culled, (entity) => {
+ disposeDecoders(world, entity);
}));
subs.push(world.onRemove(Deleted, (entity) => {
diff --git a/packages/runtime/tsconfig.json b/packages/runtime/tsconfig.json
index d3ad84ef..516eda7b 100644
--- a/packages/runtime/tsconfig.json
+++ b/packages/runtime/tsconfig.json
@@ -3,7 +3,8 @@
"compilerOptions": {
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo",
"erasableSyntaxOnly": false,
- "lib": ["es2024", "dom"]
+ "lib": ["es2024", "DOM", "DOM.Iterable"],
+ "types": ["@webgpu/types"]
},
"include": ["src"]
}