The VS Code of video editing, built for agents.
TypeScript
Update runtime package with new media handling features and dependencies
konstantin-paulus2774a07280f5
diff --git a/package-lock.json b/package-lock.jsonindex dfa6a387..90d8070c 100644--- a/package-lock.json+++ b/package-lock.json@@ -12331,11 +12331,13 @@"license": "MPL-2.0","dependencies": {"animejs": "^4.4.1",+ "colord": "^2.9.3","koota": "^0.6.6","mediabunny": "^1.32.2","sqids": "^0.3.0"},"devDependencies": {+ "@webgpu/types": "^0.1.71","typescript": "~5.9.3"}}diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.mdindex 43ea1351..2e1833a7 100644--- a/packages/runtime/MIGRATION.md+++ b/packages/runtime/MIGRATION.md@@ -32,6 +32,11 @@ Working doc for migrating the editor engine from bitecs (in `apps/web/src/compon| Actions: group, frame, clipboard | `src/actions/{group,frame,clipboard}.ts` | frame.ts also hosts `switchActiveScene` (was api/base.ts): sets the ActiveScene world trait, nothing else || Actions: assets, folders (model) | `src/actions/{assets,folders}.ts` | allocateId/raiseIdCounter, appendAssets/upsertAsset/removeAssets, getAssetFile/Blob, insertAssetInTimeline; folder tree ops return changed records for app persistence || Observers (runtime half) | `src/world/observers.ts` | koota subscriptions wiring cache upkeep, Computed mirrors, and time-range reactions; auto-registered by `createRuntimeWorld` (returns a disposer). See semantic notes |+| Media: decoders + audio bus | `src/media/*` | frame-cache, keyframe-index, time-stretcher, surface, html, audio-peaks moved verbatim; image, video, sequence, audio, shader with entity-based resolvers; audio-bus koota-ported; dispose walkers in `src/media/dispose.ts` and wired as observers |+| Media: caption family | `src/media/caption/*` | types, subtitles, utils (incl. `setChars`), position, seven presets, resolver. See semantic notes on sparse TextStyle |+| Text rendering | `src/utils/text.ts` (full port) | tokenize/shape/render joined the Token seed; reads RenderSurface.ctx + Camera; no-ops without a surface. Gradient construction split into `src/systems/gradients.ts` (render system reuses it) |+| Fonts | `src/fonts/{fixtures,utils}.ts` | WebFonts fixtures verbatim; `loadWebFont` uses the Fonts world trait, resolves the FontFaceSet via globalThis (worker-capable), guards on missing FontFace (headless), and drops db persistence (app wraps it). `getLocalFonts` + `restoreFonts` stay app-side |+| Support utils | `src/utils/{color,async}.ts`, `clamp` in math | parseColor/colorToHex/rgbToColor (colord dep), AsyncMutex/attempt || Seeds (types only, rest follows) | `src/utils/text.ts` (Token/Line/TokenOptions), `src/capture/format.ts` (ContainerFormat) | The remaining engine files merge into these on their slice |Verification style used so far: `npx tsc --noEmit` in the package, plus throwaway `smoke-test.tmp.ts` in the package root run with `npm exec --yes tsx@latest -- smoke-test.tmp.ts`, deleted afterwards (no test infra in the package yet). koota caps live worlds at 16; call `world.destroy()` in test loops.@@ -60,7 +65,13 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All- Not faithful on purpose: keyframe auto-sync on authored edits did not move into observers. koota change events don't say which field changed, and syncing every field of a trait would mint spurious keyframes on untouched properties. Editing surfaces call `syncKeyframeTrack` for the property they changed (resizeEntity already does for width/height).- Size propagation is a Size observer now (`propagateSize` on add/change): bitecs restored sizes through `resizeEntity`, koota deserialization is a plain set. `resizeEntity` no longer calls propagateSize itself.- `clampTrimToAssetDuration` now goes through `entity.set(Trim)` (fires change events) but only when actually clamping; bitecs wrote the store silently. The Trim onChange handler re-enters once through the clamp and no-ops on the second pass.-- Still app-side from `api/observers.ts`: persistence scheduling, timeline index rebuilds, `syncInteractiveState`, Selected waveform show/hide, ClipHeight/Expanded, ExportSettings defaults, drag-origin traits. Media slice: dispose hooks on Deleted/Culled onAdd.+- Still app-side from `api/observers.ts`: persistence scheduling, timeline index rebuilds, `syncInteractiveState`, Selected waveform show/hide, ClipHeight/Expanded, ExportSettings defaults, drag-origin traits.+- Media handle traits renamed with a `Handle` suffix (`VideoDecoderHandle`, `AudioBusHandle`, ...) and typed to the concrete classes via type-only imports, so the trait and the class it stores (`AudioDecoder`, `HtmlHost`, ...) share one export surface without colliding. Resolvers are entity-based (`resolveVideoDecoder(world, entity)`), read Assets/Mode world traits, and store instances with add+set on the handle trait.+- Dispose hooks are observers now: Deleted onAdd disposes decoders/hosts/buses across the tombstoned subtree, Culled onAdd disposes decoders, ChildOf add/remove disconnects audio buses, and `reactToAssetChange`'s Caption branch drops the caption decoder. The walkers (`src/media/dispose.ts`) deliberately recurse through Deleted descendants, like bitecs.+- TextStyle became all-undefined defaults: a range override carries ONLY its explicitly set fields (sparse store slot = inherit from the node), which is what the bitecs set-observer's TextRange special case enforced. Node-level fallbacks (16px/Inter/400/leading 1) live in the text renderer's accessors, restoring the bitecs `?? 1` leading semantics the earlier trait defaults had drifted from.+- Caption presets write playhead-driven text via `setChars` (raw store write, no change events) so per-frame caption content never reaches app persistence, same as the bitecs raw array writes. Preset fills/ranges go through createEntity/appendChild, so caches prime via the observers.+- DOM caveats, all lazy (nothing touches DOM at import time): HtmlHost/SurfaceHost create elements at construction (html-in-canvas is Chromium-flag-only anyway), ShaderHost needs WebGPU, `loadWebFont` no-ops without FontFace. tsconfig gained `DOM.Iterable` + `@webgpu/types`.+- `SequenceDecoder` iterates the structural `AssetDirectoryHandle` (entries with `getFile`) instead of concrete FileSystemFileHandle, so node hosts can feed frame directories.- `switchActiveScene` only sets the ActiveScene world trait; timeline buffer clearing and index rebuild stay app-side. `frameSelection` calls it once (bitecs called it redundantly inside and outside the transaction).- `insertAssetInTimeline` returns the created entity or null when no scene is active (no toast). Folder ops are sync and return the changed records (`deleteFolder` returns removed folder/asset ids) so the app can persist; `moveAssetsToFolder` updates records in place instead of re-fronting them like `saveAsset` did. Panel navigation state (current folder) stays app-side.- Decoder/host handle traits (`src/traits/media.ts`) are AoS traits typed as opaque `object` until the decoders move in; `entity.get(VideoDecoder)` returns the instance directly.@@ -79,17 +90,16 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All## Remaining work (suggested order)-1. **Media** (`src/media/`): move `engine/decoders/*` wholesale (video, audio, image, sequence, caption/, html, shader, surface, frame-cache, keyframe-index, time-stretcher, audio-peaks) plus `services/audio-bus.ts`. Then tighten the handle trait types in `src/traits/media.ts` to the concrete classes and delete the `Handle = object` placeholder. Reinstate the dispose hooks earlier slices dropped, as observers in `world/observers.ts`: Deleted onAdd → dispose decoders/hosts/bus, Culled onAdd → dispose decoders, ChildOf events → `disconnectAudioBus`, `reactToAssetChange`'s Caption branch → drop the caption decoder.-2. **Playback system**: `systems/playback.ts` (computes `Computed.localTime/start/end/duration/visibility`, drives decoders, pushes into `FramePromises`, uses AudioEngine + Mounts). Depends on media.-3. **Capture** (`src/capture/`): `engine/encode/*` (encoder, image-encoder, buffer, format, utils). Uses Mode, FramePromises, RenderSurface.-4. **Render system**: `systems/render.ts` is runtime-capable (OffscreenCanvas) and needed for headless capture; it consumes text layout (`utils/text.ts` full port: layout/measure/render join the Token seed).-5. **Fonts**: `engine/font/{utils,fixtures}.ts` (check DOM usage: document.fonts needs an injection seam or stays app-side).-6. **Remaining utils**: captions, transition, contact-sheet, audio-sync, transcode (audit each for DOM/app deps; DOM-touching ones stay app-side).-7. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.-8. **CLI**: point node capture at the runtime package.+1. **Playback system**: `systems/playback.ts` (computes `Computed.localTime/start/end/duration/visibility`, drives decoders via the media resolvers, pushes into `FramePromises`, uses AudioEngine + Mounts; `resolveAudioBus` lives here).+2. **Capture** (`src/capture/`): `engine/encode/*` (encoder, image-encoder, buffer, format, utils). Uses Mode, FramePromises, RenderSurface.+3. **Render system**: `systems/render.ts` is runtime-capable (OffscreenCanvas) and needed for headless capture; text rendering and gradients are already in (`utils/text.ts`, `systems/gradients.ts`; `addStopsTo` stayed private to gradients).+4. **Remaining utils**: captions, transition, contact-sheet, audio-sync, transcode (audit each for DOM/app deps; DOM-touching ones stay app-side).+5. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.+6. **CLI**: point node capture at the runtime package.## Layout reminders- Package exports: root `.` plus `./traits`, `./systems`, `./media`, `./capture` (see package.json). Everything currently also re-exported from root `src/index.ts`.- tsconfig sets `erasableSyntaxOnly: false` (enums with reverse lookups, same as apps/web).-- Deps: koota (patched), mediabunny, animejs, sqids (asset/folder id allocation).+- Deps: koota (patched), mediabunny, animejs, sqids (asset/folder id allocation), colord (parseColor). Dev: @webgpu/types (shader host).+- AoS handle traits accept class instances and null through plain `entity.set(trait, value)`; `getStore` returns the value array.diff --git a/packages/runtime/package.json b/packages/runtime/package.jsonindex 5149ac08..e036b811 100644--- a/packages/runtime/package.json+++ b/packages/runtime/package.json@@ -22,11 +22,13 @@],"dependencies": {"animejs": "^4.4.1",+ "colord": "^2.9.3","koota": "^0.6.6","mediabunny": "^1.32.2","sqids": "^0.3.0"},"devDependencies": {+ "@webgpu/types": "^0.1.71","typescript": "~5.9.3"}}diff --git a/packages/runtime/src/actions/timing.ts b/packages/runtime/src/actions/timing.tsindex e993045c..512abe50 100644--- a/packages/runtime/src/actions/timing.ts+++ b/packages/runtime/src/actions/timing.ts@@ -10,7 +10,7 @@ import { store } from '../world/store';import { DEFAULT_DURATION_FRAMES } from '../constants';import {Geometry, Group, Paint, Audio, Caption, Cache, Computed, Delay,- PlaybackRate, Trim,+ PlaybackRate, Trim, CaptionDecoderHandle,} from '../traits';import { isGroupLike } from '../queries/predicates';import { getParentNode } from '../queries/hierarchy';@@ -139,8 +139,12 @@ export function reactToAssetChange(world: World, entity: Entity) {} else if (entity.has(Audio)) {reactToGeometryDurationChange(world, entity);} else if (entity.has(Caption)) {- // Re-pointed transcript: the caption decoder must re-resolve with the- // new asset. Decoder disposal moves in with the media slice.+ // Re-pointed transcript: drop the decoder so it re-resolves with the+ // new asset.+ if (entity.has(CaptionDecoderHandle)) {+ entity.get(CaptionDecoderHandle)?.dispose();+ entity.set(CaptionDecoderHandle, null);+ }}}diff --git a/packages/runtime/src/assets/types.ts b/packages/runtime/src/assets/types.tsindex 8a9c81af..ef3ac168 100644--- a/packages/runtime/src/assets/types.ts+++ b/packages/runtime/src/assets/types.ts@@ -20,6 +20,8 @@ export interface AssetDirectoryHandle {export type TranscriptWord = { text: string; start: number; end: number };+export type WordGroup = TranscriptWord[];+export type Transcript = { text: string; words: TranscriptWord[] }[];interface AssetBase {diff --git a/packages/runtime/src/fonts/fixtures.ts b/packages/runtime/src/fonts/fixtures.tsnew file mode 100644index 00000000..62f7b079--- /dev/null+++ b/packages/runtime/src/fonts/fixtures.ts@@ -0,0 +1,62 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++export const WebFonts = {+ Inter: {+ weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'],+ url: 'https://fonts.gstatic.com/s/inter/v20/UcCo3FwrK3iLTcviYwYZ8UA3.woff2',+ },+ 'The Bold Font': {+ weights: ['500'],+ url: 'https://diffusion-studio-public.s3.eu-central-1.amazonaws.com/fonts/the-bold-font.ttf',+ },+ 'Komika Axis': {+ weights: ['400'],+ url: 'https://diffusion-studio-public.s3.eu-central-1.amazonaws.com/fonts/komika-axis.ttf',+ },+ Geologica: {+ weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'],+ url: 'https://fonts.gstatic.com/s/geologica/v1/oY1l8evIr7j9P3TN9YwNAdyjzUyDKkKdAGOJh1UlCDUIhAIdhCZOn1fLsig7jfvCCPHZckUWE1lELWNN-w.woff2',+ },+ Nunito: {+ weights: ['200', '300', '400', '500', '600', '700', '800', '900'],+ url: 'https://fonts.gstatic.com/s/nunito/v26/XRXV3I6Li01BKofINeaBTMnFcQ.woff2',+ },+ Figtree: {+ weights: ['300', '400', '500', '600', '700', '800', '900'],+ url: 'https://fonts.gstatic.com/s/figtree/v5/_Xms-HUzqDCFdgfMm4S9DaRvzig.woff2',+ },+ Urbanist: {+ weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'],+ url: 'https://fonts.gstatic.com/s/urbanist/v15/L0x-DF02iFML4hGCyMqlbS1miXK2.woff2',+ },+ Montserrat: {+ weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'],+ url: 'https://fonts.gstatic.com/s/montserrat/v26/JTUSjIg1_i6t8kCHKm459WlhyyTh89Y.woff2',+ },+ Bangers: {+ weights: ['400'],+ url: 'https://fonts.gstatic.com/s/bangers/v20/FeVQS0BTqb0h60ACH55Q2J5hm24.woff2',+ },+ Chewy: {+ weights: ['400'],+ url: 'https://fonts.gstatic.com/s/chewy/v18/uK_94ruUb-k-wn52KjI9OPec.woff2',+ },+ 'Source Code Pro': {+ weights: ['200', '300', '400', '500', '600', '700', '800', '900'],+ url: 'https://fonts.gstatic.com/s/sourcecodepro/v22/HI_SiYsKILxRpg3hIP6sJ7fM7PqlPevWnsUnxg.woff2',+ },+} as const;++export const FONT_WEIGHTS = {+ '100': 'Thin',+ '200': 'Extra Light',+ '300': 'Light',+ '400': 'Normal',+ '500': 'Medium',+ '600': 'Semi Bold',+ '700': 'Bold',+ '800': 'Extra Bold',+ '900': 'Black',+} as const;diff --git a/packages/runtime/src/fonts/index.ts b/packages/runtime/src/fonts/index.tsindex a8e32130..38e756ee 100644--- a/packages/runtime/src/fonts/index.ts+++ b/packages/runtime/src/fonts/index.ts@@ -1,2 +1,3 @@-// Font loading; fixtures and utils follow from engine/font/.export * from './types';+export * from './fixtures';+export * from './utils';diff --git a/packages/runtime/src/fonts/utils.ts b/packages/runtime/src/fonts/utils.tsnew file mode 100644index 00000000..64d3b4a7--- /dev/null+++ b/packages/runtime/src/fonts/utils.ts@@ -0,0 +1,97 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Web font loading (was engine/font/utils.ts, minus the app-side pieces:+// getLocalFonts needs window.queryLocalFonts, and font persistence is the+// app's IndexedDB concern; it wraps loadWebFont and re-calls it on restore).++import { WebFonts } from './fixtures';+import { FontStyle } from '../constants';+import { Fonts } from '../traits';++import type { World } from 'koota';+import type * as types from './types';++const STYLE_MAP = {+ [FontStyle.NORMAL]: 'normal',+ [FontStyle.ITALIC]: 'italic',+ [FontStyle.OBLIQUE]: 'oblique',+} as const;++/** The environment's FontFaceSet: document.fonts on a page, self.fonts in a worker. */+function getFontFaceSet(): FontFaceSet | null {+ const scope = globalThis as { document?: { fonts?: FontFaceSet }; fonts?: FontFaceSet };+ return scope.document?.fonts ?? scope.fonts ?? null;+}++/**+ * Get common web fonts+ */+export function getWebFonts(): types.FontSources[] {+ return Object.keys(WebFonts).map((family) => {+ return {+ family,+ variants: WebFonts[family as keyof typeof WebFonts].weights.map((weight) => {+ return {+ family,+ source: `url(${WebFonts[family as keyof typeof WebFonts].url})`,+ weight: weight,+ };+ }),+ };+ });+}++export async function loadWebFont(+ world: World,+ family: keyof typeof WebFonts,+ style: FontStyle = FontStyle.NORMAL,+ weight?: string,+): Promise<types.FontSource> {+ const source = `url(${WebFonts[family].url})`;+ const font: types.FontSource = {+ source,+ family,+ weight,+ style,+ };++ const fonts = world.get(Fonts)?.list;+ // No FontFace = headless environment without font rasterization; text+ // measurement falls back to the system font.+ if (!fonts || typeof FontFace === 'undefined') return font;++ // Already loaded+ if (fonts.some((f) => f.source === font.source && f.weight === font.weight)) {+ return font;+ }++ fonts.push(font);++ // Load all weights if no weight is specified+ if (!weight) {+ const config = WebFonts[family as keyof typeof WebFonts];+ const weights = config.weights;+ weight = weights.length > 1 ? `${weights[0]} ${weights[weights.length - 1]}` : weights[0];+ }++ const fontFace = new FontFace(family, source, { weight, style: STYLE_MAP[style] });++ await new Promise((resolve, reject) => {+ fontFace+ .load()+ .then((loaded) => {+ getFontFaceSet()?.add(loaded);+ resolve(null);+ })+ .catch((error) => {+ world.set(Fonts, {+ list: fonts.filter((f) => f.source !== font.source && f.weight !== font.weight),+ });+ reject(error);+ });+ });++ return font;+}diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.tsindex 0c7d0123..7c8faa44 100644--- a/packages/runtime/src/index.ts+++ b/packages/runtime/src/index.ts@@ -9,6 +9,7 @@ export * from './actions';export * from './assets/types';export * from './systems';export * from './math';+export * from './media';export * from './capture';export * from './fonts';export * from './utils';diff --git a/packages/runtime/src/math/common.ts b/packages/runtime/src/math/common.tsindex 3f13b64f..1b544b55 100644--- a/packages/runtime/src/math/common.ts+++ b/packages/runtime/src/math/common.ts@@ -16,3 +16,7 @@ export function degToRad(deg: number): number {export function radToDeg(rad: number): number {return rad * RAD_TO_DEG;}++export function clamp(value: number, min: number, max: number): number {+ return Math.min(Math.max(value, min), max);+}diff --git a/packages/runtime/src/media/audio-bus.ts b/packages/runtime/src/media/audio-bus.tsnew file mode 100644index 00000000..85d2d1bb--- /dev/null+++ b/packages/runtime/src/media/audio-bus.ts@@ -0,0 +1,71 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { store } from '../world/store';+import { Muted, Computed, AudioEngine } from '../traits';+import { attempt } from '../utils/async';+import { assert } from '../utils/assert';++import type { Entity, World } from 'koota';++/**+ * Per-entity audio bus. Each clip and each scene gets its own gain node.+ * Child buses connect directly into the parent bus's input; there is no+ * per-track sub-mix anymore (each clip is its own layer).+ */+export class AudioBus {+ public context: BaseAudioContext;++ private gain: GainNode;+ private entity: Entity;+ private world: World;+ private _input: AudioNode;++ public constructor(world: World, entity: Entity) {+ const context = world.get(AudioEngine)?.context;+ assert(context, 'World has no audio context');+ this.context = context;+ this.gain = context.createGain();+ this._input = this.gain;+ this.entity = entity;+ this.world = world;+ }++ public get input(): AudioNode {+ return this._input;+ }++ public getGain(): GainNode {+ return this.gain;+ }++ public sync(): void {+ this.gain.gain.value = this.getVolume();+ }++ public connect(node: AudioNode) {+ this.gain.connect(node);+ }++ public mute(): void {+ this.gain.gain.value = 0;+ }++ public disconnect() {+ attempt(() => this.gain.disconnect());+ this._input = this.gain;+ }++ private getVolume(): number {+ const volumeDb = store(this.world, Computed).volume[this.entity.id()] ?? 0;+ const muted = this.entity.has(Muted);++ /** Minimum dB value: treated as silence (maps to linear gain 0). */+ if (muted || volumeDb === -Infinity) {+ return 0;+ }++ return Math.pow(10, volumeDb / 20);+ }+}diff --git a/packages/runtime/src/media/audio-peaks.ts b/packages/runtime/src/media/audio-peaks.tsnew file mode 100644index 00000000..71450578--- /dev/null+++ b/packages/runtime/src/media/audio-peaks.ts@@ -0,0 +1,129 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { AudioSampleSink } from 'mediabunny';++import { getAudioTrack } from './audio';++import type { AudioAsset, VideoAsset } from '../assets/types';++const peakCache = new Map<string, Uint8ClampedArray>();+const inflight = new Map<string, Promise<Uint8ClampedArray | null>>();++const TARGET_BARS = 128;++export function clearAudioPeaksCache() {+ peakCache.clear();+ inflight.clear();+}++/**+ * Synchronous, allocation-free peak lookup for hot paths (render loop).+ * Returns cached peaks or `null`. Kicks off a build on first call.+ */+export function getAudioPeaks(+ asset: AudioAsset | VideoAsset,+): Uint8ClampedArray | null {+ const key = asset.id;++ const cached = peakCache.get(key);+ if (cached) return cached;++ if (!inflight.has(key)) {+ inflight.set(key, buildPeaks(key, asset));+ }++ return null;+}++/**+ * Async peak lookup for UI components that can await.+ * Returns peaks from cache or waits for the build to finish.+ */+export async function getAudioPeaksAsync(+ asset: AudioAsset | VideoAsset,+): Promise<Uint8ClampedArray | null> {+ const key = asset.id;++ const cached = peakCache.get(key);+ if (cached) return cached;++ let promise = inflight.get(key);+ if (!promise) {+ promise = buildPeaks(key, asset);+ inflight.set(key, promise);+ }++ return promise;+}++async function buildPeaks(+ key: string,+ asset: AudioAsset | VideoAsset,+): Promise<Uint8ClampedArray | null> {+ try {+ const track = await getAudioTrack(asset);+ if (!track) return null;++ const sink = new AudioSampleSink(track);+ const chunks: Uint8ClampedArray[] = [];++ for await (const sample of sink.samples()) {+ let floats: Float32Array;+ let numChannels: number;+ let numFrames: number;+ try {+ const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });+ floats = new Float32Array(size / Float32Array.BYTES_PER_ELEMENT);+ sample.copyTo(floats, { format: 'f32', planeIndex: 0 });++ numChannels = sample.numberOfChannels;+ numFrames = floats.length / numChannels;+ } finally {+ sample.close();+ }++ const framePeaks = new Uint8ClampedArray(numFrames);+ for (let f = 0; f < numFrames; f++) {+ let max = 0;+ for (let ch = 0; ch < numChannels; ch++) {+ max = Math.max(max, Math.pow(Math.abs(floats[f * numChannels + ch]), 0.8));+ }+ framePeaks[f] = Math.floor(max * 255);+ }+ chunks.push(framePeaks);+ }++ if (chunks.length === 0) return null;++ const totalFrames = chunks.reduce((s, c) => s + c.length, 0);+ const merged = new Uint8ClampedArray(totalFrames);+ let offset = 0;+ for (const chunk of chunks) {+ merged.set(chunk, offset);+ offset += chunk.length;+ }++ const peaks = new Uint8ClampedArray(TARGET_BARS);+ const ratio = totalFrames / TARGET_BARS;++ for (let i = 0; i < TARGET_BARS; i++) {+ const start = Math.floor(i * ratio);+ const end = Math.floor((i + 1) * ratio);+ let max = 0;+ for (let j = start; j < end && j < totalFrames; j++) {+ max = Math.max(max, merged[j]);+ }+ peaks[i] = max;+ }++ peakCache.set(key, peaks);+ return peaks;+ } catch (e) {+ console.error('Failed to build audio peaks', e);+ return null;+ } finally {+ inflight.delete(key);+ }+}diff --git a/packages/runtime/src/media/audio.ts b/packages/runtime/src/media/audio.tsnew file mode 100644index 00000000..faedff1f--- /dev/null+++ b/packages/runtime/src/media/audio.ts@@ -0,0 +1,277 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Input, ALL_FORMATS, BlobSource, AudioBufferSink, type WrappedAudioBuffer, type InputAudioTrack } from 'mediabunny';++import { AssetId, AudioDecoderHandle, Assets } from '../traits';+import { AsyncMutex } from '../utils/async';+import { assert } from '../utils/assert';+import { getAssetFile } from '../actions/assets';+import { TimeStretcher } from './time-stretcher';++import type { Entity, World } from 'koota';+import type { AudioAsset, VideoAsset } from '../assets/types';+import type { AudioBus } from './audio-bus';++const audioTrackCache = new Map<string, Promise<InputAudioTrack | null>>();++export function clearAudioTrackCache() {+ audioTrackCache.clear();+}++/**+ * Keyed by asset ID so multiple decoders for the same source+ * audio reuse a single Input + InputAudioTrack.+ */+export function getAudioTrack(source: AudioAsset | VideoAsset) {+ let promise = audioTrackCache.get(source.id);+ if (promise) return promise;++ promise = (async () => {+ try {+ const blob = await getAssetFile(source);+ const input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(blob),+ });+ return await input.getPrimaryAudioTrack();+ } catch {+ audioTrackCache.delete(source.id);+ return null;+ }+ })();++ audioTrackCache.set(source.id, promise);+ return promise;+}++type PlayOptions = {+ relativeFrom: number;+ relativeTo: number;+ trimStart: number;+ trimEnd: number;+ playbackRate: number;+ currentTime: number;+ relativeDelay: number;+}++export class AudioDecoder {+ readonly assetId: string;++ private readonly asset: AudioAsset | VideoAsset;+ private readonly mutex = new AsyncMutex();+ private readonly audioNodes = new Set<AudioBufferSourceNode>();++ private iterator: AsyncGenerator<WrappedAudioBuffer, void, unknown> | null = null;+ private firstBuffer: WrappedAudioBuffer | null = null;+ private lastBuffer: WrappedAudioBuffer | null = null;+ private stretcher: TimeStretcher | null = null;+ private nextTimestamp = 0;+ private audioTrack: InputAudioTrack | null = null;+ private sink!: AudioBufferSink;++ public ready = false;++ public constructor(asset: AudioAsset | VideoAsset) {+ this.asset = asset;+ this.assetId = asset.id;+ }++ public async init() {+ this.audioTrack = await getAudioTrack(this.asset);+ assert(this.audioTrack, 'Audio track not found');+ this.sink = new AudioBufferSink(this.audioTrack);+ this.ready = true;+ }++ private renderData(bus: AudioBus, options: PlayOptions, data: Float32Array[] | null, sampleRate: number) {+ if (!data) {+ return;+ }++ const { relativeDelay, playbackRate } = options;++ const trimStart = options.trimStart / playbackRate;+ const trimEnd = options.trimEnd / playbackRate;+ const context = bus.context;+ const input = bus.input;++ const newBuffer = new AudioBuffer({+ length: data[0]!.length,+ sampleRate,+ numberOfChannels: data.length,+ });+ for (let c = 0; c < data.length; c++) {+ newBuffer.copyToChannel(data[c] as Float32Array<ArrayBuffer>, c);+ }++ const newBufferTimestamp = this.nextTimestamp;+ this.nextTimestamp += newBuffer.duration;++ let start = relativeDelay + newBufferTimestamp;+ let offset = 0;+ let duration = newBuffer.duration;++ // Check if the buffer goes beyond the clip's end, and if so, truncate it+ if (newBufferTimestamp + duration > trimEnd) {+ duration = trimEnd - newBufferTimestamp;+ }++ // Check if the buffer starts before the clip's start, and if so, trim the start+ if (newBufferTimestamp < trimStart) {+ start += trimStart - newBufferTimestamp;+ offset += trimStart - newBufferTimestamp;+ }++ // Check if the start is before the context's current time (which is illegal)+ if (start < context.currentTime) {+ offset += context.currentTime - start;+ start = context.currentTime;+ }++ // Dumb, DIRTY hack. For some reason (probably due to some bug in the Web Audio API), sometimes a clicking noise+ // can be heard on the boundary between two audio clips. By shifting all audio buffers by half a sample, this is+ // typically avoided. My guess is that it forces the renderer into a sub-sample rendering algorithm which doesn't+ // have the buggy behavior.+ start += 0.5 / context.sampleRate;++ duration -= offset;+ duration = Math.max(0, duration);++ const bufferSource = context.createBufferSource();+ bufferSource.buffer = newBuffer;+ bufferSource.connect(input);+ bufferSource.start(start, offset, duration);++ this.audioNodes.add(bufferSource);+ bufferSource.onended = () => this.audioNodes.delete(bufferSource);+ }++ public async playTo(bus: AudioBus | null, options: PlayOptions) {+ if (!this.asset.channels || !this.asset.sampleRate || !bus) {+ // Asset has no audio data+ return;+ }++ const { relativeFrom, relativeTo } = options;++ const release = await this.mutex.acquire();++ const firstTs = this.firstBuffer?.timestamp ?? Number.POSITIVE_INFINITY;+ const lastTs = this.lastBuffer?.timestamp ?? Number.NEGATIVE_INFINITY;+ const isBufferOutOfRange = relativeFrom < firstTs || lastTs < relativeFrom - 1;++ try {+ if (this.iterator == null || this.stretcher == null || isBufferOutOfRange) {+ // Close the previous iterator so its pre-decoded AudioSamples get released.+ await this.iterator?.return();+ this.iterator = this.sink.buffers(relativeFrom);+ this.firstBuffer = null;+ this.lastBuffer = null;+ this.stretcher = null;+ }++ while (true) {+ const nextBuffer = (await this.iterator.next()).value;+ // Use the actual decoded sample rate if available+ const sampleRate = nextBuffer?.buffer.sampleRate ?? this.asset.sampleRate;+ const numberOfChannels = nextBuffer?.buffer.numberOfChannels ?? this.asset.channels;+ const playbackRate = 1 / options.playbackRate;++ this.stretcher ??= new TimeStretcher(numberOfChannels, sampleRate, playbackRate);++ if (!nextBuffer) {+ this.renderData(bus, options, this.stretcher.finalize(), sampleRate);+ break;+ }++ if (!this.firstBuffer) {+ this.nextTimestamp = nextBuffer.timestamp / options.playbackRate;+ }++ if (this.lastBuffer) {+ const sampleGap = Math.round(+ nextBuffer.timestamp - (this.lastBuffer.timestamp + this.lastBuffer.duration)+ );++ if (sampleGap > 0) {+ this.renderData(+ bus,+ options,+ Array(numberOfChannels).fill(new Float32Array(sampleGap)),+ sampleRate+ );+ }+ }++ this.firstBuffer ??= nextBuffer;+ this.lastBuffer = nextBuffer;++ const buffers: Float32Array[] = [];+ for (let c = 0; c < nextBuffer.buffer.numberOfChannels; c++) {+ const channelData = nextBuffer.buffer.getChannelData(c);+ buffers.push(channelData);+ }++ // TODO: Handle gap in timestamps case++ this.renderData(bus, options, this.stretcher.append(buffers), sampleRate);++ if (nextBuffer.timestamp >= relativeTo) {+ break;+ }+ }+ } finally {+ release();+ }+ }++ public reset() {+ if (!this.iterator) return;++ this.iterator?.return();+ this.iterator = null;+ this.firstBuffer = null;+ this.lastBuffer = null;++ for (const node of this.audioNodes) {+ node.stop();+ }+ this.audioNodes.clear();+ }+}+++type ResolvedAudioDecoder = {+ decoder: AudioDecoder;+ initPromise: Promise<void> | null;+};++export function resolveAudioDecoder(world: World, entity: Entity): ResolvedAudioDecoder | null {+ const assetId = entity.get(AssetId)?.value;+ if (!assetId) return null;++ const existing = entity.get(AudioDecoderHandle);+ if (existing && existing.assetId === assetId) {+ return {+ decoder: existing,+ initPromise: null,+ };+ }++ // Asset changed: reset old decoder and create a new one.+ if (existing) existing.reset();++ const asset = world.get(Assets)?.get(assetId);+ if (!asset || (asset.type !== 'AUDIO' && asset.type !== 'VIDEO')) return null;+ if (asset.type === 'VIDEO' && !asset.channels) return null;++ const decoder = new AudioDecoder(asset);+ entity.add(AudioDecoderHandle);+ entity.set(AudioDecoderHandle, decoder);+ return {+ decoder,+ initPromise: decoder.init(),+ };+}diff --git a/packages/runtime/src/media/caption/cascade.ts b/packages/runtime/src/media/caption/cascade.tsnew file mode 100644index 00000000..33d9ab37--- /dev/null+++ b/packages/runtime/src/media/caption/cascade.ts@@ -0,0 +1,101 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';+import { Paint, Color, TextStyle } from '../../traits';+import { renderText } from '../../utils/text';+import { loadWebFont } from '../../fonts/utils';+import { groupBy, findActiveGroup, resolveTranscript, setChars } from './utils';+import { placeCaption } from './position';+import { createEntity } from '../../actions/entities';+import { appendChild } from '../../actions/hierarchy';++import type { Entity, World } from 'koota';+import type { Asset } from '../../assets/types';+import type { CaptionDecoder } from './types';++const WIDTH = 800;+const HEIGHT = 200;++export class CascadeCaptionDecoder implements CaptionDecoder {+ public readonly type = CaptionType.CASCADE;+ public groups: ReturnType<typeof groupBy> = [];+ public ready = false;++ private readonly asset: Asset;+ private currentText = '';++ constructor(asset: Asset) {+ this.asset = asset;+ this.init();+ }++ private async init() {+ if (this.ready) return;+ const transcript = await resolveTranscript(this.asset);+ this.groups = groupBy(transcript, { length: 50 });+ this.ready = true;+ }++ public reposition(world: World, entity: Entity): boolean {+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, x: 100, defaultAlign: CaptionAlign.BOTTOM });+ }++ public applyStyles(world: World, entity: Entity): void {+ if (!this.reposition(world, entity)) return;++ entity.add(TextStyle);+ entity.set(TextStyle, {+ fontFamily: 'Inter',+ fontWeight: '300',+ fontSize: 50,+ textAlign: TextAlign.LEFT,+ textBaseline: TextBaseline.TOP,+ fontStyle: FontStyle.NORMAL,+ textCase: TextCase.ORIGINAL,+ leading: 1.2,+ });++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: 0xFFFFFF });+ appendChild(world, fill, entity);++ loadWebFont(world, 'Inter');+ }++ public seekTo(world: World, entity: Entity, relativeTime: number): void {+ const groupIndex = findActiveGroup(this.groups, relativeTime);++ if (groupIndex === -1) {+ setChars(world, entity, '');+ this.currentText = '';+ return;+ }++ const group = this.groups[groupIndex]!;++ // Progressive reveal: only show words that have started+ const text = group+ .filter(word => word.start <= relativeTime)+ .map(w => w.text)+ .join(' ');++ if (text !== this.currentText) {+ this.currentText = text;+ setChars(world, entity, text);+ }+ }++ public draw(world: World, entity: Entity): void {+ renderText(world, entity);+ }++ public dispose(): void {+ this.groups = [];+ this.currentText = '';+ }+}diff --git a/packages/runtime/src/media/caption/classic.ts b/packages/runtime/src/media/caption/classic.tsnew file mode 100644index 00000000..12d5fd12--- /dev/null+++ b/packages/runtime/src/media/caption/classic.ts@@ -0,0 +1,111 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';+import { Paint, Color, Shadow, Appearance, Blur, Offset, TextStyle } from '../../traits';+import { renderText } from '../../utils/text';+import { loadWebFont } from '../../fonts/utils';+import { groupBy, findActiveGroup, resolveTranscript, setChars } from './utils';+import { placeCaption } from './position';+import { createEntity } from '../../actions/entities';+import { appendChild } from '../../actions/hierarchy';++import type { Entity, World } from 'koota';+import type { Asset } from '../../assets/types';+import type { CaptionDecoder } from './types';++export const CLASSIC_PRESET_WIDTH = 600;+export const CLASSIC_PRESET_HEIGHT = 100;++export class ClassicCaptionDecoder implements CaptionDecoder {+ public readonly type = CaptionType.CLASSIC;+ public groups: ReturnType<typeof groupBy> = [];+ public ready = false;++ private readonly asset: Asset;+ private currentGroupIndex = -1;++ constructor(asset: Asset) {+ this.asset = asset;+ this.init();+ }++ private async init() {+ if (this.ready) return;+ const transcript = await resolveTranscript(this.asset);+ this.groups = groupBy(transcript, { duration: 0.2 });+ this.ready = true;+ }++ public reposition(world: World, entity: Entity): boolean {+ return placeCaption(world, entity, {+ width: CLASSIC_PRESET_WIDTH,+ height: CLASSIC_PRESET_HEIGHT,+ defaultAlign: CaptionAlign.CENTER,+ });+ }++ public applyStyles(world: World, entity: Entity): void {+ if (!this.reposition(world, entity)) return;++ entity.add(TextStyle);+ entity.set(TextStyle, {+ fontFamily: 'Urbanist',+ fontWeight: '600',+ fontSize: 62,+ textAlign: TextAlign.CENTER,+ textBaseline: TextBaseline.MIDDLE,+ textCase: TextCase.LOWER,+ fontStyle: FontStyle.NORMAL,+ leading: 1,+ });++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: 0xFFFFFF });+ appendChild(world, fill, entity);++ const shadow = createEntity(world);+ shadow.add(Shadow);+ shadow.add(Color);+ shadow.set(Color, { value: 0x000000 });+ shadow.add(Appearance);+ shadow.set(Appearance, { opacity: 1 });+ shadow.add(Blur);+ shadow.set(Blur, { value: 28 });+ shadow.add(Offset);+ shadow.set(Offset, { x: 0, y: 5 });+ appendChild(world, shadow, entity);++ loadWebFont(world, 'Urbanist');+ }++ public seekTo(world: World, entity: Entity, relativeTime: number): void {+ const groupIndex = findActiveGroup(this.groups, relativeTime);++ if (groupIndex === -1) {+ setChars(world, entity, '');+ this.currentGroupIndex = -1;+ return;+ }++ const group = this.groups[groupIndex]!;++ if (groupIndex !== this.currentGroupIndex) {+ this.currentGroupIndex = groupIndex;+ setChars(world, entity, group.map(w => w.text).join(' '));+ }+ }++ public draw(world: World, entity: Entity): void {+ renderText(world, entity);+ }++ public dispose(): void {+ this.groups = [];+ this.currentGroupIndex = -1;+ }+}diff --git a/packages/runtime/src/media/caption/guinea.ts b/packages/runtime/src/media/caption/guinea.tsnew file mode 100644index 00000000..69c3504d--- /dev/null+++ b/packages/runtime/src/media/caption/guinea.ts@@ -0,0 +1,155 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { store } from '../../world/store';+import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';+import { Paint, Color, Caption, ItemIndex, TextRange, TextStyle } from '../../traits';+import { renderText } from '../../utils/text';+import { loadWebFont } from '../../fonts/utils';+import { groupBy, findActiveGroup, splitSequence, clearTextRanges, resolveTranscript, setChars } from './utils';+import { placeCaption } from './position';+import { createEntity } from '../../actions/entities';+import { appendChild } from '../../actions/hierarchy';++import type { Entity, World } from 'koota';+import type { Asset } from '../../assets/types';+import type { CaptionDecoder } from './types';++const WIDTH = 700;+const HEIGHT = 200;++const HIGHLIGHT_COLOR_0 = 0xF55353;+const HIGHLIGHT_COLOR_1 = 0xFEB139;+const HIGHLIGHT_COLOR_2 = 0xF6F54D;++const PSEUDO_RANDOM_SEQUENCE = [0, 1, 0, 2, 1, 1, 0, 0, 1, 0, 2, 1, 0, 2, 1, 0, 2, 1];++export class GuineaCaptionDecoder implements CaptionDecoder {+ public readonly type = CaptionType.GUINEA;+ public groups: ReturnType<typeof groupBy> = [];+ public ready = false;++ private readonly asset: Asset;+ private currentGroupIndex = -1;+ private activeSplit: 'left' | 'right' | null = null;+ private fill: Entity | null = null;+ private colorIndex = 0;++ constructor(asset: Asset) {+ this.asset = asset;+ this.init();+ }++ private async init() {+ if (this.ready) return;+ const transcript = await resolveTranscript(this.asset);+ this.groups = groupBy(transcript, { length: 18 });+ this.ready = true;+ }++ public reposition(world: World, entity: Entity): boolean {+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.CENTER });+ }++ public applyStyles(world: World, entity: Entity): void {+ if (!this.reposition(world, entity)) return;++ entity.add(TextStyle);+ entity.set(TextStyle, {+ fontFamily: 'The Bold Font',+ fontWeight: '500',+ fontSize: 62,+ textAlign: TextAlign.CENTER,+ textBaseline: TextBaseline.MIDDLE,+ textCase: TextCase.UPPER,+ fontStyle: FontStyle.NORMAL,+ leading: 1,+ });++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: 0xFFFFFF });+ appendChild(world, fill, entity);++ loadWebFont(world, 'The Bold Font');+ }++ public seekTo(world: World, entity: Entity, relativeTime: number): void {+ const groupIndex = findActiveGroup(this.groups, relativeTime);++ if (groupIndex === -1) {+ setChars(world, entity, '');+ clearTextRanges(world, entity);+ this.currentGroupIndex = -1;+ this.activeSplit = null;+ return;+ }++ const group = this.groups[groupIndex]!;+ const [left, right] = splitSequence(group);+ const leftText = left.map(w => w.text).join(' ');+ const rightText = right.map(w => w.text).join(' ');+ const text = rightText ? `${leftText}\n${rightText}` : leftText;+ const activeSplit = right.length > 0 && relativeTime >= right[0]!.start ? 'right' : 'left';++ if (groupIndex !== this.currentGroupIndex || activeSplit !== this.activeSplit) {+ this.currentGroupIndex = groupIndex;+ this.activeSplit = activeSplit;+ setChars(world, entity, text);++ // Active line gets a random highlight color and a slightly larger font (1.1x).+ clearTextRanges(world, entity);+ const start = activeSplit === 'left' ? 0 : leftText.length + 1;+ const end = activeSplit === 'left' ? leftText.length : text.length;+ if (end > start) {+ const baseSize = store(world, TextStyle).fontSize[entity.id()] ?? 62;+ const fontSize = Math.round(baseSize * 1.1);++ const range = createEntity(world);+ range.add(TextRange);+ range.set(TextRange, { start, end });+ range.add(TextStyle);+ range.set(TextStyle, { fontSize });+ appendChild(world, range, entity);++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: HIGHLIGHT_COLOR_0 });+ fill.add(ItemIndex);+ fill.set(ItemIndex, { value: 0 });+ appendChild(world, fill, range);+ this.fill = fill;++ this.colorIndex++;+ }+ }+ }++ public draw(world: World, entity: Entity): void {+ const authored = entity.get(Caption)?.colors;+ const colors = [+ authored?.[0] ?? HIGHLIGHT_COLOR_0,+ authored?.[1] ?? HIGHLIGHT_COLOR_1,+ authored?.[2] ?? HIGHLIGHT_COLOR_2,+ ] as const;++ const index = PSEUDO_RANDOM_SEQUENCE[this.colorIndex % PSEUDO_RANDOM_SEQUENCE.length]!;++ if (this.fill) {+ store(world, Color).value[this.fill.id()] = colors[index]!;+ }++ renderText(world, entity);+ }++ public dispose(): void {+ this.groups = [];+ this.currentGroupIndex = -1;+ this.activeSplit = null;+ }+}diff --git a/packages/runtime/src/media/caption/index.ts b/packages/runtime/src/media/caption/index.tsnew file mode 100644index 00000000..457608ca--- /dev/null+++ b/packages/runtime/src/media/caption/index.ts@@ -0,0 +1,81 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { CaptionType } from '../../constants';+import { AssetId, Caption, TextStyle, CaptionDecoderHandle, Assets } from '../../traits';+import { ClassicCaptionDecoder } from './classic';+import { CascadeCaptionDecoder } from './cascade';+import { SpotlightCaptionDecoder } from './spotlight';+import { WhisperCaptionDecoder } from './whisper';+import { PaperCaptionDecoder } from './paper';+import { GuineaCaptionDecoder } from './guinea';+import { StarkCaptionDecoder } from './stark';++import type { Entity, World } from 'koota';+import type { Asset } from '../../assets/types';+import type { CaptionDecoder } from './types';++export type { CaptionDecoder } from './types';+export { ClassicCaptionDecoder, CLASSIC_PRESET_WIDTH, CLASSIC_PRESET_HEIGHT } from './classic';+export { CascadeCaptionDecoder } from './cascade';+export { SpotlightCaptionDecoder } from './spotlight';+export { WhisperCaptionDecoder } from './whisper';+export { PaperCaptionDecoder } from './paper';+export { GuineaCaptionDecoder } from './guinea';+export { StarkCaptionDecoder } from './stark';+export * from './position';+export * from './subtitles';+export * from './utils';++function createCaptionDecoder(type: CaptionType, asset: Asset): CaptionDecoder {+ switch (type) {+ case CaptionType.CLASSIC:+ return new ClassicCaptionDecoder(asset);+ case CaptionType.CASCADE:+ return new CascadeCaptionDecoder(asset);+ case CaptionType.SPOTLIGHT:+ return new SpotlightCaptionDecoder(asset);+ case CaptionType.WHISPER:+ return new WhisperCaptionDecoder(asset);+ case CaptionType.PAPER:+ return new PaperCaptionDecoder(asset);+ case CaptionType.GUINEA:+ return new GuineaCaptionDecoder(asset);+ case CaptionType.STARK:+ return new StarkCaptionDecoder(asset);+ default:+ return new ClassicCaptionDecoder(asset);+ }+}++/**+ * Lazily resolve (or create) a caption decoder for a caption entity.+ * Recreates the decoder when the caption type changes.+ * Returns null if the transcript asset isn't available yet.+ */+export function resolveCaptionDecoder(world: World, entity: Entity): CaptionDecoder | null {+ const assetId = entity.get(AssetId)?.value;+ if (!assetId) return null;++ const captionType = entity.get(Caption)?.type ?? CaptionType.CLASSIC;+ const existing = entity.get(CaptionDecoderHandle);++ if (existing && captionType === existing.type) return existing;++ const typeChanged = existing != null;+ existing?.dispose();++ const asset = world.get(Assets)?.get(assetId);+ if (!asset) return null;++ const decoder = createCaptionDecoder(captionType, asset);+ entity.add(CaptionDecoderHandle);+ entity.set(CaptionDecoderHandle, decoder);++ if (typeChanged || !entity.has(TextStyle)) {+ decoder.applyStyles(world, entity);+ }++ return decoder;+}diff --git a/packages/runtime/src/media/caption/paper.ts b/packages/runtime/src/media/caption/paper.tsnew file mode 100644index 00000000..cedceb5b--- /dev/null+++ b/packages/runtime/src/media/caption/paper.ts@@ -0,0 +1,119 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';+import { Paint, Color, TextRange, TextStyle } from '../../traits';+import { renderText } from '../../utils/text';+import { loadWebFont } from '../../fonts/utils';+import { groupBy, findActiveGroup, splitSequence, clearTextRanges, resolveTranscript, setChars } from './utils';+import { placeCaption } from './position';+import { createEntity } from '../../actions/entities';+import { appendChild } from '../../actions/hierarchy';++import type { Entity, World } from 'koota';+import type { Asset } from '../../assets/types';+import type { CaptionDecoder } from './types';++const WIDTH = 700;+const HEIGHT = 200;++export class PaperCaptionDecoder implements CaptionDecoder {+ public readonly type = CaptionType.PAPER;+ public groups: ReturnType<typeof groupBy> = [];+ public ready = false;++ private readonly asset: Asset;+ private currentGroupIndex = -1;+ private activeSplit: 'left' | 'right' | null = null;++ constructor(asset: Asset) {+ this.asset = asset;+ this.init();+ }++ private async init() {+ if (this.ready) return;+ const transcript = await resolveTranscript(this.asset);+ this.groups = groupBy(transcript, { length: 18 });+ this.ready = true;+ }++ public reposition(world: World, entity: Entity): boolean {+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.CENTER });+ }++ public applyStyles(world: World, entity: Entity): void {+ if (!this.reposition(world, entity)) return;++ entity.add(TextStyle);+ entity.set(TextStyle, {+ fontFamily: 'Montserrat',+ fontWeight: '300',+ fontSize: 50,+ textAlign: TextAlign.CENTER,+ textBaseline: TextBaseline.MIDDLE,+ textCase: TextCase.ORIGINAL,+ leading: 0.9,+ fontStyle: FontStyle.NORMAL,+ });++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: 0xFFFFFF });+ appendChild(world, fill, entity);++ loadWebFont(world, 'Montserrat');+ }++ public seekTo(world: World, entity: Entity, relativeTime: number): void {+ const groupIndex = findActiveGroup(this.groups, relativeTime);++ if (groupIndex === -1) {+ setChars(world, entity, '');+ clearTextRanges(world, entity);+ this.currentGroupIndex = -1;+ this.activeSplit = null;+ return;+ }++ const group = this.groups[groupIndex]!;+ const [left, right] = splitSequence(group);+ const leftText = left.map(w => w.text).join(' ');+ const rightText = right.map(w => w.text).join(' ');+ const text = rightText ? `${leftText}\n${rightText}` : leftText;+ const activeSplit = right.length > 0 && relativeTime >= right[0]!.start ? 'right' : 'left';++ if (groupIndex !== this.currentGroupIndex || activeSplit !== this.activeSplit) {+ this.currentGroupIndex = groupIndex;+ this.activeSplit = activeSplit;+ setChars(world, entity, text);++ // Two-line display with newline separator.+ // The active line gets weight '500', inactive stays at the node's '300'.+ clearTextRanges(world, entity);+ const start = activeSplit === 'left' ? 0 : leftText.length + 1;+ const end = activeSplit === 'left' ? leftText.length : text.length;+ if (end > start) {+ const range = createEntity(world);+ range.add(TextRange);+ range.set(TextRange, { start, end });+ range.add(TextStyle);+ range.set(TextStyle, { fontWeight: '500' });+ appendChild(world, range, entity);+ }+ }+ }++ public draw(world: World, entity: Entity): void {+ renderText(world, entity);+ }++ public dispose(): void {+ this.groups = [];+ this.currentGroupIndex = -1;+ this.activeSplit = null;+ }+}diff --git a/packages/runtime/src/media/caption/position.ts b/packages/runtime/src/media/caption/position.tsnew file mode 100644index 00000000..9d9be1c6--- /dev/null+++ b/packages/runtime/src/media/caption/position.ts@@ -0,0 +1,49 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { store } from '../../world/store';+import { CaptionAlign } from '../../constants';+import { Caption, Position, Computed } from '../../traits';+import { getParentNode } from '../../queries/hierarchy';+import { resizeEntity } from '../../actions/resize';++import type { Entity, World } from 'koota';++// Margin between the caption box and the parent edge for top/bottom alignment.+export const CAPTION_MARGIN = 100;++/**+ * Sizes the caption box and places it within its parent. Horizontal placement+ * belongs to the preset (centered unless a fixed `x` is given); vertical+ * placement follows the entity's `Caption.verticalAlign`, falling back to the+ * preset's default. Returns false when the entity has no parent to place+ * against.+ */+export function placeCaption(+ world: World,+ entity: Entity,+ preset: { width: number; height: number; x?: number; defaultAlign: CaptionAlign },+): boolean {+ const parent = getParentNode(entity);+ if (parent === null) return false;+ const computed = store(world, Computed);+ const parentWidth = computed.width[parent.id()]!;+ const parentHeight = computed.height[parent.id()]!;++ resizeEntity(world, entity, { width: preset.width, height: preset.height });++ const align = entity.get(Caption)?.verticalAlign ?? preset.defaultAlign;+ const y = align === CaptionAlign.TOP+ ? CAPTION_MARGIN+ : align === CaptionAlign.BOTTOM+ ? parentHeight - preset.height - CAPTION_MARGIN+ : (parentHeight - preset.height) / 2;++ entity.add(Position);+ entity.set(Position, {+ x: preset.x ?? (parentWidth - preset.width) / 2,+ y,+ });+ return true;+}diff --git a/packages/runtime/src/media/caption/spotlight.ts b/packages/runtime/src/media/caption/spotlight.tsnew file mode 100644index 00000000..c90ffd25--- /dev/null+++ b/packages/runtime/src/media/caption/spotlight.ts@@ -0,0 +1,137 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { store } from '../../world/store';+import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';+import { Paint, Color, Caption, TextRange, TextStyle } from '../../traits';+import { renderText } from '../../utils/text';+import { loadWebFont } from '../../fonts/utils';+import { groupBy, findActiveGroup, clearTextRanges, resolveTranscript, setChars } from './utils';+import { placeCaption } from './position';+import { createEntity } from '../../actions/entities';+import { appendChild } from '../../actions/hierarchy';++import type { Entity, World } from 'koota';+import type { Asset } from '../../assets/types';+import type { CaptionDecoder } from './types';+++const WIDTH = 700;+const HEIGHT = 100;+const HIGHLIGHT_COLOR = 0x24D5FF;++export class SpotlightCaptionDecoder implements CaptionDecoder {+ public readonly type = CaptionType.SPOTLIGHT;+ public groups: ReturnType<typeof groupBy> = [];+ public ready = false;++ private readonly asset: Asset;+ private currentGroupIndex = -1;+ private currentWordIndex = -1;+ private fill: Entity | null = null;++ constructor(asset: Asset) {+ this.asset = asset;+ this.init();+ }++ private async init() {+ if (this.ready) return;+ const transcript = await resolveTranscript(this.asset);+ this.groups = groupBy(transcript, { length: 10 });+ this.ready = true;+ }++ public reposition(world: World, entity: Entity): boolean {+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.CENTER });+ }++ public applyStyles(world: World, entity: Entity): void {+ if (!this.reposition(world, entity)) return;++ entity.add(TextStyle);+ entity.set(TextStyle, {+ fontFamily: 'The Bold Font',+ fontWeight: '500',+ fontStyle: FontStyle.ITALIC,+ fontSize: 70,+ textAlign: TextAlign.CENTER,+ textBaseline: TextBaseline.MIDDLE,+ textCase: TextCase.ORIGINAL,+ leading: 1,+ });++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: 0xFFFFFF });+ appendChild(world, fill, entity);++ loadWebFont(world, 'The Bold Font');+ }++ public seekTo(world: World, entity: Entity, relativeTime: number): void {+ const groupIndex = findActiveGroup(this.groups, relativeTime);++ if (groupIndex === -1) {+ setChars(world, entity, '');+ clearTextRanges(world, entity);+ this.fill = null;+ this.currentGroupIndex = -1;+ this.currentWordIndex = -1;+ return;+ }++ const group = this.groups[groupIndex]!;++ const wordIndex = group.findIndex(word =>+ relativeTime >= word.start && relativeTime <= word.end+ );++ const text = group.map(w => w.text).join(' ');++ if (groupIndex !== this.currentGroupIndex || wordIndex !== this.currentWordIndex) {+ this.currentGroupIndex = groupIndex;+ this.currentWordIndex = wordIndex;+ setChars(world, entity, text);++ clearTextRanges(world, entity);+ this.fill = null;++ if (group.length > 1 && wordIndex !== -1) {+ const start = group.slice(0, wordIndex).map(w => w.text).join(' ').length;+ const end = group.slice(0, wordIndex + 1).map(w => w.text).join(' ').length;+ const range = createEntity(world);+ range.add(TextRange);+ range.set(TextRange, { start, end });+ appendChild(world, range, entity);++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: HIGHLIGHT_COLOR });+ appendChild(world, fill, range);+ this.fill = fill;+ }+ }+ }++ public draw(world: World, entity: Entity): void {+ const highlightColor = entity.get(Caption)?.colors?.[0] ?? HIGHLIGHT_COLOR;++ if (this.fill) {+ store(world, Color).value[this.fill.id()] = highlightColor;+ }++ renderText(world, entity);+ }++ public dispose(): void {+ this.groups = [];+ this.currentGroupIndex = -1;+ this.currentWordIndex = -1;+ }+}diff --git a/packages/runtime/src/media/caption/stark.ts b/packages/runtime/src/media/caption/stark.tsnew file mode 100644index 00000000..4714973e--- /dev/null+++ b/packages/runtime/src/media/caption/stark.ts@@ -0,0 +1,95 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { BlendMode, CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';+import { Paint, Color, Appearance, TextStyle } from '../../traits';+import { renderText } from '../../utils/text';+import { loadWebFont } from '../../fonts/utils';+import { groupBy, findActiveGroup, resolveTranscript, setChars } from './utils';+import { placeCaption } from './position';+import { createEntity } from '../../actions/entities';+import { appendChild } from '../../actions/hierarchy';++import type { Entity, World } from 'koota';+import type { Asset } from '../../assets/types';+import type { CaptionDecoder } from './types';++const WIDTH = 700;+const HEIGHT = 100;++export class StarkCaptionDecoder implements CaptionDecoder {+ public readonly type = CaptionType.STARK;+ public groups: ReturnType<typeof groupBy> = [];+ public ready = false;++ private readonly asset: Asset;+ private currentGroupIndex = -1;++ constructor(asset: Asset) {+ this.asset = asset;+ this.init();+ }++ private async init() {+ if (this.ready) return;+ const transcript = await resolveTranscript(this.asset);+ this.groups = groupBy(transcript, { duration: 0.2 });+ this.ready = true;+ }++ public reposition(world: World, entity: Entity): boolean {+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.CENTER });+ }++ public applyStyles(world: World, entity: Entity): void {+ if (!this.reposition(world, entity)) return;++ entity.add(TextStyle);+ entity.set(TextStyle, {+ fontFamily: 'Figtree',+ fontWeight: '800',+ fontSize: 70,+ textAlign: TextAlign.CENTER,+ textBaseline: TextBaseline.MIDDLE,+ textCase: TextCase.UPPER,+ fontStyle: FontStyle.NORMAL,+ leading: 1,+ });++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: 0xFFFFFF });+ fill.add(Appearance);+ fill.set(Appearance, { blendMode: BlendMode.DIFFERENCE });+ appendChild(world, fill, entity);++ loadWebFont(world, 'Figtree');+ }++ public seekTo(world: World, entity: Entity, relativeTime: number): void {+ const groupIndex = findActiveGroup(this.groups, relativeTime);++ if (groupIndex === -1) {+ setChars(world, entity, '');+ this.currentGroupIndex = -1;+ return;+ }++ if (groupIndex !== this.currentGroupIndex) {+ this.currentGroupIndex = groupIndex;+ setChars(world, entity, this.groups[groupIndex]!.map(w => w.text).join(' '));+ }+ }++ public draw(world: World, entity: Entity): void {+ renderText(world, entity);+ }++ public dispose(): void {+ this.groups = [];+ this.currentGroupIndex = -1;+ }+}diff --git a/packages/runtime/src/media/caption/subtitles.ts b/packages/runtime/src/media/caption/subtitles.tsnew file mode 100644index 00000000..8f9ab2a2--- /dev/null+++ b/packages/runtime/src/media/caption/subtitles.ts@@ -0,0 +1,60 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { Transcript } from '../../assets/types';++const TIMESTAMP_RE = /(?:\d{1,2}:)?\d{1,2}:\d{2}[.,]\d{1,3}/g;++function parseTimestamp(raw: string): number {+ return raw+ .replace(',', '.')+ .split(':')+ .reduce((acc, part) => acc * 60 + Number(part), 0);+}++/**+ * Parses SRT or WebVTT text into a `Transcript`. Each cue becomes one segment+ * (a sentence boundary for grouping). Cues carry no word timings, so each+ * word's window is synthesized proportional to its length within the cue.+ */+export function parseSubtitles(text: string): Transcript {+ const transcript: Transcript = [];+ const blocks = text.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n').split(/\n{2,}/);++ for (const block of blocks) {+ const lines = block.split('\n');+ const timingIndex = lines.findIndex((line) => line.includes('-->'));+ if (timingIndex === -1) continue; // WEBVTT header, NOTE/STYLE/REGION blocks, stray indices++ const timestamps = lines[timingIndex].match(TIMESTAMP_RE);+ if (!timestamps || timestamps.length < 2) continue;+ const start = parseTimestamp(timestamps[0]);+ const end = parseTimestamp(timestamps[1]);+ if (!(end > start)) continue;++ // Strip inline markup: HTML/VTT tags (<i>, <c.class>, <00:01.000>) and+ // SSA-style overrides ({\an8}).+ const content = lines+ .slice(timingIndex + 1)+ .join(' ')+ .replace(/<[^>]*>/g, '')+ .replace(/\{[^}]*\}/g, '');++ const tokens = content.split(/\s+/).filter(Boolean);+ if (tokens.length === 0) continue;++ const total = tokens.reduce((acc, token) => acc + token.length, 0);+ const duration = end - start;+ let elapsed = 0;+ const words = tokens.map((token) => {+ const wordStart = start + (elapsed / total) * duration;+ elapsed += token.length;+ return { text: token, start: wordStart, end: start + (elapsed / total) * duration };+ });++ transcript.push({ text: tokens.join(' '), words });+ }++ return transcript.sort((a, b) => a.words[0].start - b.words[0].start);+}diff --git a/packages/runtime/src/media/caption/types.ts b/packages/runtime/src/media/caption/types.tsnew file mode 100644index 00000000..c4f3a0e6--- /dev/null+++ b/packages/runtime/src/media/caption/types.ts@@ -0,0 +1,18 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { Entity, World } from 'koota';+import type { CaptionType } from '../../constants';+import type { WordGroup } from '../../assets/types';++export interface CaptionDecoder {+ readonly type: CaptionType;+ groups: WordGroup[];+ ready: boolean;+ applyStyles(world: World, entity: Entity): void;+ reposition(world: World, entity: Entity): boolean;+ seekTo(world: World, entity: Entity, relativeTime: number): void;+ draw(world: World, entity: Entity): void;+ dispose(): void;+}diff --git a/packages/runtime/src/media/caption/utils.ts b/packages/runtime/src/media/caption/utils.tsnew file mode 100644index 00000000..57627a0d--- /dev/null+++ b/packages/runtime/src/media/caption/utils.ts@@ -0,0 +1,133 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { store } from '../../world/store';+import { ChildOf, TextRange, Cache, Chars } from '../../traits';+import { deleteEntity } from '../../actions/entities';+import { getAssetFile } from '../../actions/assets';+import { parseSubtitles } from './subtitles';++import type { Entity, World } from 'koota';+import type { Asset, Transcript, WordGroup } from '../../assets/types';++export type TextRangeOverride = {+ start: number;+ end: number;+};+++export type GroupByOptions = {+ duration: number;+} | {+ length: number;+};++const ENDING_PUNCTUATION = ['.', '!', '?', ',', ';'];++export function groupBy(transcript: Transcript, options: GroupByOptions): WordGroup[] {+ const groups: WordGroup[] = [[]];++ for (const segment of transcript) {+ // Segment boundaries are sentence boundaries — never merge across them.+ if (groups[groups.length - 1]!.length > 0) {+ groups.push([]);+ }++ for (const word of segment.words) {+ const last = groups[groups.length - 1]!;+ const charCount = last.reduce((acc, w) => acc + w.text.length, 0);+ const duration = last.reduce((acc, w) => acc + (w.end - w.start), 0);++ if ('duration' in options && duration + (word.end - word.start) > options.duration) {+ groups.push([]);+ } else if ('length' in options && charCount + word.text.length > options.length) {+ groups.push([]);+ }++ groups.at(-1)?.push(word);++ if (ENDING_PUNCTUATION.some(p => word.text.endsWith(p))) {+ groups.push([]);+ }+ }+ }++ return groups.filter(g => g.length > 0);+}++/**+ * Find the active group index for a given time.+ * Returns -1 if no group is active.+ */+export function findActiveGroup(groups: WordGroup[], relativeTime: number): number {+ return groups.findIndex(group =>+ relativeTime >= group[0]!.start && relativeTime <= (group.at(-1)?.end ?? 0)+ );+}++/**+ * Playhead-driven caption text: a raw store write, no change events, so the+ * app never persists per-frame caption content (bitecs wrote the array too).+ */+export function setChars(world: World, entity: Entity, text: string) {+ if (!entity.has(Chars)) entity.add(Chars);+ store(world, Chars).value[entity.id()] = text;+}++export function clearTextRanges(world: World, parent: Entity) {+ for (const range of world.query(ChildOf(parent), TextRange)) {+ deleteEntity(world, range);+ }++ if (parent.has(Cache)) {+ store(world, Cache).textRanges[parent.id()] = [];+ }+}++/**+ * Split a word group into two halves at the nearest word boundary+ * to the midpoint of the joined text. Used by paper and guinea presets.+ */+export function splitSequence(sequence: WordGroup): [WordGroup, WordGroup] {+ const text = sequence.map(w => w.text).join(' ');+ const midPoint = Math.ceil(text.length / 2);++ let index = text.length;+ for (let i = midPoint, j = midPoint; i > 0 && j < text.length - 1; i--, j++) {+ if (text[i] === ' ') {+ index = i;+ break;+ }+ if (text[j] === ' ') {+ index = j;+ break;+ }+ }++ const leftText = text.slice(0, index).trim();+ const splitAt = leftText.split(/ /).length;++ return [sequence.slice(0, splitAt), sequence.slice(splitAt)];+}+++/**+ * Resolve a transcript from any asset type that can carry one.+ */+export async function resolveTranscript(asset: Asset): Promise<Transcript> {+ if (asset.type === 'TRANSCRIPT') {+ const file = await getAssetFile(asset);+ const text = await file.text();+ if (asset.mimeType === 'application/x-subrip' || asset.mimeType === 'text/vtt') {+ return parseSubtitles(text);+ }+ return JSON.parse(text);+ }++ if ((asset.type === 'AUDIO' || asset.type === 'VIDEO') && asset.transcript) {+ return asset.transcript;+ }++ return [];+}diff --git a/packages/runtime/src/media/caption/whisper.ts b/packages/runtime/src/media/caption/whisper.tsnew file mode 100644index 00000000..1ed00959--- /dev/null+++ b/packages/runtime/src/media/caption/whisper.ts@@ -0,0 +1,94 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { CaptionAlign, CaptionType, PaintType, FontStyle, TextAlign, TextBaseline, TextCase } from '../../constants';+import { Paint, Color, TextStyle } from '../../traits';+import { renderText } from '../../utils/text';+import { loadWebFont } from '../../fonts/utils';+import { groupBy, findActiveGroup, resolveTranscript, setChars } from './utils';+import { placeCaption } from './position';+import { createEntity } from '../../actions/entities';+import { appendChild } from '../../actions/hierarchy';++import type { Entity, World } from 'koota';+import type { Asset } from '../../assets/types';+import type { CaptionDecoder } from './types';++const WIDTH = 1000;+const HEIGHT = 100;++export class WhisperCaptionDecoder implements CaptionDecoder {+ public readonly type = CaptionType.WHISPER;+ public groups: ReturnType<typeof groupBy> = [];+ public ready = false;++ private readonly asset: Asset;+ private currentGroupIndex = -1;++ constructor(asset: Asset) {+ this.asset = asset;+ this.init();+ }++ private async init() {+ if (this.ready) return;+ const transcript = await resolveTranscript(this.asset);+ this.groups = groupBy(transcript, { duration: 2 });++ this.ready = true;+ }++ public reposition(world: World, entity: Entity): boolean {+ return placeCaption(world, entity, { width: WIDTH, height: HEIGHT, defaultAlign: CaptionAlign.BOTTOM });+ }++ public applyStyles(world: World, entity: Entity): void {+ if (!this.reposition(world, entity)) return;++ entity.add(TextStyle);+ entity.set(TextStyle, {+ fontFamily: 'Montserrat',+ fontWeight: '400',+ fontSize: 40,+ textAlign: TextAlign.CENTER,+ textBaseline: TextBaseline.MIDDLE,+ textCase: TextCase.ORIGINAL,+ leading: 1.4,+ fontStyle: FontStyle.NORMAL,+ });++ const fill = createEntity(world);+ fill.add(Paint);+ fill.set(Paint, { value: PaintType.SOLID });+ fill.add(Color);+ fill.set(Color, { value: 0xFFFFFF });+ appendChild(world, fill, entity);++ loadWebFont(world, 'Montserrat');+ }++ public seekTo(world: World, entity: Entity, relativeTime: number): void {+ const groupIndex = findActiveGroup(this.groups, relativeTime);++ if (groupIndex === -1) {+ setChars(world, entity, '');+ this.currentGroupIndex = -1;+ return;+ }++ if (groupIndex !== this.currentGroupIndex) {+ this.currentGroupIndex = groupIndex;+ setChars(world, entity, this.groups[groupIndex]!.map(w => w.text).join(' '));+ }+ }++ public draw(world: World, entity: Entity): void {+ renderText(world, entity);+ }++ public dispose(): void {+ this.groups = [];+ this.currentGroupIndex = -1;+ }+}diff --git a/packages/runtime/src/media/dispose.ts b/packages/runtime/src/media/dispose.tsnew file mode 100644index 00000000..1e890e2b--- /dev/null+++ b/packages/runtime/src/media/dispose.ts@@ -0,0 +1,87 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Subtree walkers releasing live runtime handles (was part of api/utils.ts).+// They deliberately include Deleted descendants: a tombstoned subtree keeps+// its records for undo, but its decoders and buses must not linger.++import {+ ChildOf,+ ImageDecoderHandle, VideoDecoderHandle, SequenceDecoderHandle,+ AudioDecoderHandle, CaptionDecoderHandle,+ HtmlHostHandle, SurfaceHostHandle, ShaderHostHandle, AudioBusHandle,+} from '../traits';++import type { Entity, World } from 'koota';++export function disposeDecoders(world: World, entity: Entity): void {+ if (entity.has(ImageDecoderHandle)) {+ entity.get(ImageDecoderHandle)?.dispose();+ entity.set(ImageDecoderHandle, null);+ }+ if (entity.has(VideoDecoderHandle)) {+ entity.get(VideoDecoderHandle)?.dispose();+ entity.set(VideoDecoderHandle, null);+ }+ if (entity.has(SequenceDecoderHandle)) {+ entity.get(SequenceDecoderHandle)?.dispose();+ entity.set(SequenceDecoderHandle, null);+ }+ if (entity.has(AudioDecoderHandle)) {+ entity.get(AudioDecoderHandle)?.reset();+ entity.set(AudioDecoderHandle, null);+ }+ if (entity.has(CaptionDecoderHandle)) {+ entity.get(CaptionDecoderHandle)?.dispose();+ entity.set(CaptionDecoderHandle, null);+ }++ for (const child of world.query(ChildOf(entity))) {+ disposeDecoders(world, child);+ }+}++export function disposeHtmlHosts(world: World, entity: Entity): void {+ if (entity.has(HtmlHostHandle)) {+ entity.get(HtmlHostHandle)?.dispose();+ entity.set(HtmlHostHandle, null);+ }++ for (const child of world.query(ChildOf(entity))) {+ disposeHtmlHosts(world, child);+ }+}++export function disposeSurfaceHosts(world: World, entity: Entity): void {+ if (entity.has(SurfaceHostHandle)) {+ entity.get(SurfaceHostHandle)?.dispose();+ entity.set(SurfaceHostHandle, null);+ }++ for (const child of world.query(ChildOf(entity))) {+ disposeSurfaceHosts(world, child);+ }+}++export function disposeShaderHosts(world: World, entity: Entity): void {+ if (entity.has(ShaderHostHandle)) {+ entity.get(ShaderHostHandle)?.dispose();+ entity.set(ShaderHostHandle, null);+ }++ for (const child of world.query(ChildOf(entity))) {+ disposeShaderHosts(world, child);+ }+}++export function disconnectAudioBus(world: World, entity: Entity): void {+ if (entity.has(AudioBusHandle)) {+ entity.get(AudioBusHandle)?.disconnect();+ entity.set(AudioBusHandle, null);+ }++ for (const child of world.query(ChildOf(entity))) {+ disconnectAudioBus(world, child);+ }+}diff --git a/packages/runtime/src/media/frame-cache.ts b/packages/runtime/src/media/frame-cache.tsnew file mode 100644index 00000000..7630b459--- /dev/null+++ b/packages/runtime/src/media/frame-cache.ts@@ -0,0 +1,186 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++type FrameCacheConfig = {+ pixels: number;+ count: number;+}++type FrameTile = {+ tileIndex: number;+ frameIndex: number;+}++type Frame = VideoFrame | ImageBitmap;++function getDisplaySize(frame: Frame) {+ if (frame instanceof VideoFrame) {+ return [frame.displayWidth, frame.displayHeight];+ }+ return [frame.width, frame.height]+}++export class FrameCache {+ public readonly config: FrameCacheConfig;+ public readonly atlas = new OffscreenCanvas(0, 0);+ public readonly atlasCtx = this.atlas.getContext('2d')!;+ public readonly source = new OffscreenCanvas(0, 0);+ public readonly sourceCtx = this.source.getContext('2d')!;++ private tileWidth: number = 0;+ private tileHeight: number = 0;+ private columns: number = 0;+ private tiles: FrameTile[];++ public rotation: number = 0;+ public leftFrameIndex: number;+ public rightFrameIndex: number;+ public lastInserted: number = -1;++ public constructor(config: FrameCacheConfig) {+ this.config = config;+ this.leftFrameIndex = 0;+ this.rightFrameIndex = config.count - 1;+ this.tiles = [];+ }++ private resizeCaches(frame: Frame) {+ const rotation = this.rotation;+ const [width, height] = getDisplaySize(frame);+ const pixelCount = width * height;++ const factor = Math.min(1, Math.sqrt(this.config.pixels / pixelCount));++ const sourceWidth = rotation === 90 || rotation === 270+ ? height+ : width;++ const sourceHeight = rotation === 90 || rotation === 270+ ? width+ : height;++ this.tileWidth = Math.floor(sourceWidth * factor);+ this.tileHeight = Math.floor(sourceHeight * factor);++ // Lay tiles in a 2D grid+ this.columns = Math.ceil(Math.sqrt(this.config.count));+ const rows = Math.ceil(this.config.count / this.columns);++ const absoluteWidth = this.tileWidth * this.columns;+ const absoluteHeight = this.tileHeight * rows;++ if (this.atlas.width !== absoluteWidth || this.atlas.height !== absoluteHeight) {+ this.atlas.width = absoluteWidth;+ this.atlas.height = absoluteHeight;+ this.atlasCtx.imageSmoothingQuality = 'high';+ this.tiles = []; // setting the canvas size resets the tiles+ }+ }++ private drawRotated(frame: Frame) {+ const ctx = this.sourceCtx;+ const rotation = this.rotation;++ const [displayWidth, displayHeight] = getDisplaySize(frame);++ const width = rotation === 90 || rotation === 270+ ? displayHeight+ : displayWidth;++ const height = rotation === 90 || rotation === 270+ ? displayWidth+ : displayHeight;++ if (this.source.width !== width || this.source.height !== height) {+ this.source.width = width;+ this.source.height = height;+ ctx.imageSmoothingEnabled = false;+ }++ ctx.resetTransform();+ ctx.clearRect(0, 0, width, height);+ ctx.translate(width / 2, height / 2);+ ctx.rotate((rotation * Math.PI) / 180);+ ctx.drawImage(frame, -displayWidth / 2, -displayHeight / 2, displayWidth, displayHeight);+ }++ private evictTiles() {+ if (this.tiles.length < this.config.count) return;++ this.tiles = this.tiles.filter(t => t.frameIndex >= this.leftFrameIndex && t.frameIndex <= this.rightFrameIndex);+ }++ private insertTile(frameIndex: number) {+ let tileIndex = 0;+ while (this.tiles.some(t => t.tileIndex === tileIndex)) {+ tileIndex++;+ }++ this.tiles.push({ tileIndex, frameIndex });++ const ctx = this.atlasCtx;+ const x = (tileIndex % this.columns) * this.tileWidth;+ const y = Math.floor(tileIndex / this.columns) * this.tileHeight;++ ctx.clearRect(x, y, this.tileWidth, this.tileHeight);+ ctx.drawImage(this.source, x, y, this.tileWidth, this.tileHeight);+ }++ public has(frameIndex: number) {+ return this.tiles.some(t => t.frameIndex === frameIndex);+ }++ /**+ * Frame index of the cached frame closest to `frameIndex`, or `undefined` when+ * nothing is cached within `tolerance`. Ties resolve to the earlier frame, so a+ * playhead sitting between two cached frames never shows one that hasn't played yet.+ */+ public findNearest(frameIndex: number, tolerance: number): number | undefined {+ let best: number | undefined;+ let bestDistance = tolerance + 1;++ for (const tile of this.tiles) {+ const distance = Math.abs(tile.frameIndex - frameIndex);+ if (distance < bestDistance || (best !== undefined && distance === bestDistance && tile.frameIndex < best)) {+ bestDistance = distance;+ best = tile.frameIndex;+ }+ }++ return best;+ }++ public insert(frame: VideoFrame | ImageBitmap, index: number) {+ if (this.has(index)) return;++ this.evictTiles();+ this.resizeCaches(frame);+ this.drawRotated(frame);+ this.insertTile(index);+ this.lastInserted = index;+ }++ public findTile(frameIndex: number) {+ const tile = this.tiles.find(t => t.frameIndex === frameIndex);++ if (!tile) return;++ const x = (tile.tileIndex % this.columns) * this.tileWidth;+ const y = Math.floor(tile.tileIndex / this.columns) * this.tileHeight;++ return { x, y, width: this.tileWidth, height: this.tileHeight };+ }++ public dispose() {+ this.atlas.width = 0;+ this.atlas.height = 0;+ this.source.width = 0;+ this.source.height = 0;+ this.tileWidth = 0;+ this.tileHeight = 0;+ this.columns = 0;+ this.lastInserted = -1;+ this.tiles = [];+ }+}diff --git a/packages/runtime/src/media/html.ts b/packages/runtime/src/media/html.tsnew file mode 100644index 00000000..ec4de380--- /dev/null+++ b/packages/runtime/src/media/html.ts@@ -0,0 +1,152 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;++// html-in-canvas (https://github.com/WICG/html-in-canvas). Chromium only,+// behind chrome://flags/#canvas-draw-element; the API surface is still+// moving, so every touchpoint is typed and isolated here.+type DrawElementContext = Ctx2D & {+ drawElementImage(source: Element | unknown, dx: number, dy: number, dw: number, dh: number): DOMMatrix;+};++export function isHtmlInCanvasSupported(): boolean {+ return typeof CanvasRenderingContext2D !== 'undefined'+ && 'drawElementImage' in CanvasRenderingContext2D.prototype;+}++/**+ * Resolves after the browser's next rendering update (style/layout/paint) has completed.+ */+export function nextRenderingUpdate(): Promise<void> {+ return new Promise(resolve => {+ // timeout in case requestAnimationFrame is not called+ const timeout = setTimeout(resolve, 250);++ requestAnimationFrame(() => {+ clearTimeout(timeout);+ setTimeout(resolve, 0);+ });+ });+}++let htmlCanvas: HTMLCanvasElement | null = null;+let htmlCtx: CanvasRenderingContext2D | null = null;+let htmlHosts = 0;++function acquireLayoutCanvas(): HTMLCanvasElement {+ if (!htmlCanvas) {+ htmlCanvas = document.createElement('canvas');+ htmlCanvas.width = 1;+ htmlCanvas.height = 1;+ htmlCanvas.toggleAttribute('layoutsubtree', true);+ htmlCanvas.style.cssText = 'position:fixed;left:0;top:0;width:1px;height:1px;pointer-events:none;';+ document.body.appendChild(htmlCanvas);+ htmlCtx = htmlCanvas.getContext('2d')!;+ }+ htmlHosts++;+ return htmlCanvas;+}++function releaseLayoutCanvas(): void {+ if (--htmlHosts === 0) {+ htmlCanvas?.remove();+ htmlCanvas = null;+ htmlCtx = null;+ }+}++/**+ * DOM host for one HTML paint: a div inside the shared layoutsubtree canvas.+ * The mount document inserts the paint's JSX children into `element` as real,+ * reactive DOM; the browser lays them out at the parent geometry's box size+ * and `draw` rasterizes the result at the destination's device scale and+ * blits it into the rendering context (stage or offscreen alike).+ */+export class HtmlHost {+ public readonly element: HTMLDivElement;+ private disposed = false;+ private warned = false;+ private width = -1;+ private height = -1;++ public constructor() {+ const el = document.createElement('div');+ // pointer-events: none — layoutsubtree children participate in hit+ // testing and would otherwise swallow page interactions.+ el.style.cssText = 'position:absolute;left:0;top:0;overflow:hidden;pointer-events:none;';+ acquireLayoutCanvas().appendChild(el);+ this.element = el;+ }++ /**+ * Resolves asynchronously loaded html resources.+ */+ public whenReady(timeInSeconds: number): Promise<void> {+ // sync animations+ for (const animation of this.element.getAnimations({ subtree: true })) {+ if (animation.playState !== 'paused') animation.pause();+ animation.currentTime = timeInSeconds * 1000;+ }++ // wait for fonts to be ready+ const pending: Promise<unknown>[] = [document.fonts.ready];++ // wait for images to be ready+ for (const image of this.element.querySelectorAll('img')) {+ // `complete` covers loaded, failed, and srcless images alike; the+ // rest are still in flight, and a failed decode just paints nothing.+ if (image.complete) continue;+ pending.push(image.decode().catch(() => undefined));+ }++ return Promise.all(pending).then(() => nextRenderingUpdate());+ }++ public setSize(width: number, height: number): void {+ if (width === this.width && height === this.height) return;+ this.width = width;+ this.height = height;+ this.element.style.width = `${width}px`;+ this.element.style.height = `${height}px`;+ }++ public draw(ctx: Ctx2D, width: number, height: number): void {+ if (this.disposed || !htmlCanvas || !htmlCtx) return;++ this.setSize(width, height);++ try {+ const m = ctx.getTransform();+ const sx = Math.hypot(m.a, m.b) || 1;+ const sy = Math.hypot(m.c, m.d) || 1;+ const pw = Math.max(1, Math.ceil(width * sx));+ const ph = Math.max(1, Math.ceil(height * sy));++ // Grow-only: shrinking would clear and reallocate every time hosts+ // of different sizes share the canvas within one frame.+ if (htmlCanvas.width < pw) htmlCanvas.width = pw;+ if (htmlCanvas.height < ph) htmlCanvas.height = ph;++ htmlCtx.setTransform(sx, 0, 0, sy, 0, 0);+ htmlCtx.clearRect(0, 0, width, height);+ (htmlCtx as DrawElementContext).drawElementImage(this.element, 0, 0, width, height);+ ctx.drawImage(htmlCanvas, 0, 0, pw, ph, 0, 0, width, height);+ } catch (e) {+ // A transient failure (subtree not yet laid out or painted) just+ // skips the frame; log once so a permanent one doesn't spam.+ if (!this.warned) {+ this.warned = true;+ console.error(`Error drawing <HtmlPaint> content: ${e instanceof Error ? `${e.name}: ${e.message}` : e}`);+ }+ }+ }++ public dispose(): void {+ if (this.disposed) return;+ this.disposed = true;+ this.element.remove();+ releaseLayoutCanvas();+ }+}diff --git a/packages/runtime/src/media/image.ts b/packages/runtime/src/media/image.tsnew file mode 100644index 00000000..4bd45a3f--- /dev/null+++ b/packages/runtime/src/media/image.ts@@ -0,0 +1,258 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { AssetId, ImageDecoderHandle, Assets } from '../traits';+import { getAssetFile } from '../actions/assets';++import type { Entity, World } from 'koota';+import type { ImageAsset } from '../assets/types';++export type DecodedImage = ImageBitmap | HTMLImageElement;++export interface ImageDecoder {+ readonly assetId: string;+ readonly failed: boolean;+ readonly ready: boolean;+ init(): Promise<void>;+ getBitmap(targetWidth: number, targetHeight: number): DecodedImage | null;+ dispose(): void;+}++/**+ * Image decoder backed by `createImageBitmap`. Used for raster formats+ * (PNG, JPEG, WebP, …) where we want a GPU-friendly bitmap.+ *+ * Fetching + decoding starts immediately on construction.+ */+export class BitmapImageDecoder implements ImageDecoder {+ readonly assetId: string;+ public failed = false;+ private bitmap: ImageBitmap | null = null;+ private disposed = false;++ public constructor(private readonly asset: ImageAsset) {+ this.assetId = asset.id;+ }++ public async init() {+ if (this.disposed) return;++ try {+ const file = await getAssetFile(this.asset);+ const result = await createImageBitmap(file);+ if (this.disposed) {+ result.close();+ return;+ }+ this.bitmap = result;+ } catch (e) {+ console.error('Error decoding image', e);+ this.failed = true;+ }+ }+++ public get ready() {+ return this.bitmap !== null;+ }++ public getBitmap(_targetWidth: number, _targetHeight: number): ImageBitmap | null {+ // TODO: mipmap — pick the LOD closest to targetWidth × targetHeight+ return this.bitmap;+ }++ public dispose() {+ if (this.disposed) return;+ if (this.bitmap) {+ this.bitmap.close();+ this.bitmap = null;+ }+ this.disposed = true;+ }+}++/**+ * Image decoder backed by `HTMLImageElement`. Used for formats that+ * `createImageBitmap` doesn't handle reliably across browsers — most+ * notably SVG, where we want the browser's native vector renderer.+ */+export class ElementImageDecoder implements ImageDecoder {+ readonly assetId: string;+ public failed = false;+ private element: HTMLImageElement | null = null;+ private url: string | null = null;+ private disposed = false;+ private readonly width: number;+ private readonly height: number;++ public constructor(private readonly asset: ImageAsset) {+ this.assetId = asset.id;+ this.width = asset.width;+ this.height = asset.height;+ }++ public get ready() {+ return this.element !== null;+ }++ public async init() {+ if (this.disposed) return;++ try {+ const file = await getAssetFile(this.asset);+ const url = URL.createObjectURL(file);++ await new Promise<void>((resolve) => {+ const img = new Image();+ img.onload = () => {+ if (this.disposed) {+ URL.revokeObjectURL(url);+ return resolve();+ }+ // SVGs without explicit width/height attributes report+ // naturalWidth/Height = 0. Pin to the asset's recorded+ // dimensions so getScaledImageProps gets a real aspect.+ img.width = this.width;+ img.height = this.height;+ this.element = img;+ this.url = url;+ resolve();+ };+ img.onerror = () => {+ URL.revokeObjectURL(url);+ this.failed = true;+ resolve();+ };+ img.src = url;+ })+ } catch (e) {+ console.error('Error decoding image', e);+ this.failed = true;+ }+ }++ public getBitmap(_targetWidth: number, _targetHeight: number): HTMLImageElement | null {+ return this.element;+ }++ public dispose() {+ if (this.disposed) return;+ if (this.url) {+ URL.revokeObjectURL(this.url);+ this.url = null;+ }+ this.element = null;+ this.disposed = true;+ }+}++function createImageDecoder(asset: ImageAsset): ImageDecoder {+ if (asset.mimeType === 'image/svg+xml') {+ return new ElementImageDecoder(asset);+ }+ return new BitmapImageDecoder(asset);+}++type DecoderCacheEntry = {+ decoder: ImageDecoder;+ initPromise: Promise<void>;+ refs: number;+};++/** One decoded bitmap per asset id, scoped per world. */+type ImageDecoderCache = Map<string, DecoderCacheEntry>;++const decoderCaches = new WeakMap<World, ImageDecoderCache>();++function getDecoderCache(world: World): ImageDecoderCache {+ let cache = decoderCaches.get(world);+ if (!cache) {+ cache = new Map();+ decoderCaches.set(world, cache);+ }+ return cache;+}++/**+ * Per-entity handle onto a shared, refcounted decoder. Duplicated entities+ * pointing at the same asset share one decode; disposing a handle releases+ * one reference and the underlying decoder is disposed with the last one.+ */+class SharedImageDecoder implements ImageDecoder {+ private released = false;++ public constructor(+ private readonly entry: DecoderCacheEntry,+ private readonly cache: ImageDecoderCache,+ ) { }++ public get assetId() {+ return this.entry.decoder.assetId;+ }+ public get failed() {+ return this.entry.decoder.failed;+ }+ public get ready() {+ return this.entry.decoder.ready;+ }++ public init(): Promise<void> {+ return this.entry.initPromise;+ }++ public getBitmap(targetWidth: number, targetHeight: number): DecodedImage | null {+ return this.entry.decoder.getBitmap(targetWidth, targetHeight);+ }++ public dispose() {+ if (this.released) return;+ this.released = true;+ this.entry.refs--;+ if (this.entry.refs === 0) {+ this.cache.delete(this.entry.decoder.assetId);+ this.entry.decoder.dispose();+ }+ }+}++type ResolvedImageDecoder = {+ decoder: ImageDecoder;+ initPromise: Promise<void> | null;+}++export function resolveImageDecoder(world: World, entity: Entity): ResolvedImageDecoder | null {+ const assetId = entity.get(AssetId)?.value;+ if (!assetId) return null;++ const existing = entity.get(ImageDecoderHandle);+ if (existing && existing.assetId === assetId) {+ return {+ decoder: existing,+ initPromise: null,+ };+ }++ // Asset changed: release the old decoder and acquire one for the new asset.+ if (existing) existing.dispose();++ const asset = world.get(Assets)?.get(assetId);+ if (!asset || asset.type !== 'IMAGE') return null;++ const cache = getDecoderCache(world);+ let entry = cache.get(assetId);+ if (!entry) {+ const decoder = createImageDecoder(asset);+ entry = { decoder, initPromise: decoder.init(), refs: 0 };+ cache.set(assetId, entry);+ }+ entry.refs++;++ const handle = new SharedImageDecoder(entry, cache);+ entity.add(ImageDecoderHandle);+ entity.set(ImageDecoderHandle, handle);+ return {+ decoder: handle,+ initPromise: entry.initPromise,+ };+}diff --git a/packages/runtime/src/media/index.ts b/packages/runtime/src/media/index.tsindex 06882413..86367259 100644--- a/packages/runtime/src/media/index.ts+++ b/packages/runtime/src/media/index.ts@@ -1 +1,18 @@-// Media decoding: video, audio, image, sequence, caption/, html, shader, surface, frame-cache, time-stretcher (moves wholesale from engine/decoders/).+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++export * from './audio';+export * from './audio-bus';+export * from './audio-peaks';+export * from './caption';+export * from './dispose';+export * from './frame-cache';+export * from './html';+export * from './image';+export * from './keyframe-index';+export * from './sequence';+export * from './shader';+export * from './surface';+export * from './time-stretcher';+export * from './video';diff --git a/packages/runtime/src/media/keyframe-index.ts b/packages/runtime/src/media/keyframe-index.tsnew file mode 100644index 00000000..d046fa5f--- /dev/null+++ b/packages/runtime/src/media/keyframe-index.ts@@ -0,0 +1,109 @@+/**+ * This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.+ */++import { EncodedPacketSink } from 'mediabunny';++import type { InputVideoTrack } from 'mediabunny';++/**+ * Key packets walked between two yields to the event loop. Keeps an all-intra+ * track (every packet is a keyframe) from starving the render loop.+ */+const YIELD_INTERVAL = 512;++const indexCache = new Map<string, KeyframeIndex>();++export function clearKeyframeIndexCache() {+ indexCache.clear();+}++/**+ * Returns the shared keyframe index for an asset, starting the walk on first+ * access. Cheap to call repeatedly — one index is built per asset, not per decoder.+ */+export function getKeyframeIndex(assetId: string, track: InputVideoTrack): KeyframeIndex {+ let index = indexCache.get(assetId);++ if (!index) {+ index = new KeyframeIndex(track);+ indexCache.set(assetId, index);+ }++ return index;+}++/**+ * Ascending presentation timestamps of every key packet in a video track, walked+ * ahead of time so seeking resolves its seed keyframe with a binary search+ * instead of a container lookup.+ */+export class KeyframeIndex {+ private timestamps: number[] = [];+ private complete = false;++ public constructor(track: InputVideoTrack) {+ this.build(track);+ }++ private async build(track: InputVideoTrack) {+ try {+ const sink = new EncodedPacketSink(track);++ let packet = await sink.getFirstKeyPacket({ metadataOnly: true });+ let walked = 0;++ while (packet) {+ this.timestamps.push(packet.timestamp);++ // `getNextKeyPacket` resolves without I/O once the sample table is parsed,+ // so awaiting it only queues a microtask and never lets the browser paint.+ // A timer is a macrotask, which does.+ if (++walked % YIELD_INTERVAL === 0) {+ await new Promise((resolve) => setTimeout(resolve));+ }++ packet = await sink.getNextKeyPacket(packet, { metadataOnly: true });+ }++ this.complete = true;+ } catch (e) {+ // A disposed input (project closed, permission revoked) lands here; the+ // index simply stays incomplete and callers fall back to a container lookup.+ console.error('Error building keyframe index', e);+ }+ }++ /**+ * Timestamp of the last keyframe at or before `seconds`, or `null` when the+ * index cannot answer: `seconds` precedes the first keyframe, or the walk has+ * not reached that far yet and a closer keyframe may still show up.+ */+ public floor(seconds: number): number | null {+ const timestamps = this.timestamps;+ const length = timestamps.length;++ if (length === 0 || seconds < timestamps[0]) {+ return null;+ }+ if (!this.complete && seconds > timestamps[length - 1]) {+ return null;+ }++ let low = 0;+ let high = length - 1;++ while (low < high) {+ const mid = (low + high + 1) >> 1;+ if (timestamps[mid] <= seconds) {+ low = mid;+ } else {+ high = mid - 1;+ }+ }++ return timestamps[low];+ }+}diff --git a/packages/runtime/src/media/sequence.ts b/packages/runtime/src/media/sequence.tsnew file mode 100644index 00000000..a646a475--- /dev/null+++ b/packages/runtime/src/media/sequence.ts@@ -0,0 +1,266 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { AssetId, SequenceDecoderHandle, Assets } from '../traits';+import { assert } from '../utils/assert';+import { FrameCache } from './frame-cache';++import type { Entity, World } from 'koota';+import type { SequenceAsset } from '../assets/types';++const FRAME_EXTENSIONS = /\.(png|jpe?g|webp|avif|bmp|gif)$/i;++export type SequenceBufferMode = 'discarded' | 'idle' | 'alive';+const IDLE_TIMEOUT_MS = 10_000;++export class SequenceDecoder {+ public currentFrame: number = 0;+ public errored = false;+ public disposed = false;+ public hasCache = true;+ public asset: SequenceAsset;+ public mode: SequenceBufferMode = 'alive';+ public initialized: Promise<void>;++ private frames: File[] = [];+ private lastRenderedFrame: number = -1;+ private seekGeneration = 0;+ private idleTimer: ReturnType<typeof setTimeout> | null = null;+ private seekLock: Promise<void> = Promise.resolve();++ public readonly cache = new FrameCache({ pixels: 720 * 720, count: 49 });+ public readonly canvas = new OffscreenCanvas(0, 0);+ public readonly ctx = this.canvas.getContext('2d')!;++ public constructor(asset: SequenceAsset, hasCache: boolean) {+ this.hasCache = hasCache;+ this.asset = asset;+ this.initialized = this.initialize();+ }++ private async initialize() {+ try {+ const entries: { name: string; getFile: () => Promise<File> }[] = [];++ for await (const [name, entry] of this.asset.directoryHandle.entries()) {+ if (entry.kind !== 'file' || typeof entry.getFile !== 'function') continue;+ if (!FRAME_EXTENSIONS.test(name)) continue;+ entries.push({ name, getFile: entry.getFile.bind(entry) });+ }++ // Natural-order sort so frame_2 comes before frame_10.+ entries.sort((a, b) => a.name.localeCompare(b.name, undefined, { numeric: true }));+ this.frames = await Promise.all(entries.map(e => e.getFile()));+ assert(this.frames.length > 0, 'No frames found');+ } catch {+ this.errored = true;+ }+ }++ private updateBitmap(frameIndex: number) {+ if (frameIndex === this.lastRenderedFrame) return;+ const tile = this.cache.findTile(frameIndex);+ if (!tile) return;+ const ctx = this.ctx;++ if (this.canvas.width !== tile.width || this.canvas.height !== tile.height) {+ this.canvas.width = tile.width;+ this.canvas.height = tile.height;+ ctx.imageSmoothingEnabled = false;+ }++ ctx.clearRect(0, 0, tile.width, tile.height);+ ctx.drawImage(+ this.cache.atlas,+ tile.x,+ tile.y,+ tile.width,+ tile.height,+ 0,+ 0,+ tile.width,+ tile.height,+ );++ this.lastRenderedFrame = frameIndex;+ }++ public toBitmap() {+ if (this.canvas.width === 0 || this.canvas.height === 0) {+ return null;+ }++ return this.canvas;+ }++ private computeWindow(targetFrame: number, forward: boolean): [number, number] {+ const span = this.cache.config.count - 2;+ const lastFrameIndex = this.frames.length - 1;++ // Whole video fits in the cache — keep all of it.+ if (lastFrameIndex <= span) {+ return [0, lastFrameIndex];+ }++ const ahead = Math.round((span * 2) / 3);+ const behind = span - ahead;++ let left = targetFrame - (forward ? behind : ahead);+ let right = targetFrame + (forward ? ahead : behind);++ // Push budget that falls outside the boundaries onto the other side.+ if (left < 0) {+ right -= left;+ left = 0;+ }+ if (right > lastFrameIndex) {+ left -= right - lastFrameIndex;+ right = lastFrameIndex;+ }++ return [Math.max(0, left), Math.min(lastFrameIndex, right)];+ }++ private async updateBitmapAsync(frame: number) {+ try {+ await this.initialized;+ const file = this.frames[frame];+ if (file === undefined) return;+ const bitmap = await createImageBitmap(file);++ if (this.canvas.width !== bitmap.width || this.canvas.height !== bitmap.height) {+ this.canvas.width = bitmap.width;+ this.canvas.height = bitmap.height;+ this.ctx.imageSmoothingEnabled = false;+ }++ this.ctx.clearRect(0, 0, bitmap.width, bitmap.height);+ this.ctx.drawImage(bitmap, 0, 0, bitmap.width, bitmap.height);+ bitmap.close();+ } catch { /** ignore */ }+ }++ public async seekTo(frame: number, frameRate: number) {+ const targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);++ // Path without cache+ if (!this.hasCache) {+ return this.updateBitmapAsync(targetFrame);+ }++ this.updateBitmap(targetFrame);++ const isCurrentFrame = targetFrame === this.currentFrame;+ const isEmpty = this.frames.length === 0;+ const isIdle = this.mode === 'idle' && isCurrentFrame;+ const isCached = this.cache.has(targetFrame) && isCurrentFrame;+ const errored = this.errored;++ if (isEmpty || isCached || isIdle || errored) {+ return;+ }++ this.touch();++ const generation = ++this.seekGeneration;++ // When the cache does not have the target frame, forward must be true (latency optimization)+ const forward = this.cache.has(targetFrame) ? targetFrame >= this.currentFrame : true;+ this.currentFrame = targetFrame;++ const [left, right] = this.computeWindow(targetFrame, forward);+ this.cache.leftFrameIndex = left;+ this.cache.rightFrameIndex = right;++ let run: Promise<void>+ if (forward) {+ run = this.seekLock.then(() => this.fillCache([targetFrame, this.cache.rightFrameIndex], generation));+ } else {+ // +1 to include the target frame+ run = this.seekLock.then(() => this.fillCache([this.cache.leftFrameIndex, targetFrame], generation));+ }++ // Run after the previous seek finishes — never concurrently.+ this.seekLock = run.catch(() => { });+ }++ private async fillCache(range: [number, number], generation: number): Promise<void> {+ if (generation !== this.seekGeneration || range[0] > range[1]) return;++ for (let index = range[0]; index <= range[1]; index++) {+ const file = this.frames[index];+ if (file === undefined || this.cache.has(index)) continue;++ try {+ const bitmap = await createImageBitmap(file);+ this.cache.insert(bitmap, index);+ bitmap.close();+ this.updateBitmap(this.currentFrame);+ } catch { /** ignore */ }++ if (generation !== this.seekGeneration) break;+ }+ }++ private touch() {+ this.mode = 'alive';+ if (this.idleTimer !== null) {+ clearTimeout(this.idleTimer);+ }+ this.idleTimer = setTimeout(() => this.idle(), IDLE_TIMEOUT_MS);+ }++ public idle() {+ if (this.mode !== 'alive') return;+ this.mode = 'idle';++ if (this.idleTimer !== null) {+ clearTimeout(this.idleTimer);+ this.idleTimer = null;+ }++ this.cache.dispose();+ }+++ public dispose() {+ if (this.mode === 'discarded') return;+ this.mode = 'discarded';++ if (this.idleTimer !== null) {+ clearTimeout(this.idleTimer);+ this.idleTimer = null;+ }++ this.cache.dispose();++ // Release the display canvas backing store.+ this.canvas.width = 0;+ this.canvas.height = 0;+ this.lastRenderedFrame = -1;+ }+}++export function resolveSequenceDecoder(world: World, entity: Entity, hasCache = true): SequenceDecoder | null {+ const assetId = entity.get(AssetId)?.value;+ if (!assetId) return null;++ const asset = world.get(Assets)?.get(assetId);+ if (!asset || asset.type !== 'SEQUENCE') return null;++ // Identity check (not just id) so decoders rebuild when asset metadata —+ // e.g. frameRate — changes via `assets.save({ ...asset, frameRate: ... })`.+ const existing = entity.get(SequenceDecoderHandle);+ if (existing && existing.asset === asset) {+ existing.hasCache = hasCache;+ return existing;+ }++ existing?.dispose();++ const decoder = new SequenceDecoder(asset, hasCache);+ entity.add(SequenceDecoderHandle);+ entity.set(SequenceDecoderHandle, decoder);+ return decoder;+}diff --git a/packages/runtime/src/media/shader.ts b/packages/runtime/src/media/shader.tsnew file mode 100644index 00000000..3fe637ab--- /dev/null+++ b/packages/runtime/src/media/shader.ts@@ -0,0 +1,373 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Shader, ShaderHostHandle } from "../traits";+import { parseColor } from "../utils/color";++import type { Entity, World } from "koota";++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;+type UniformValue = number | number[] | string;++/**+ * Runtime pipeline for one shader paint: compiles the paint's fragment-stage+ * WGSL against the engine prelude, samples the media paint directly below the+ * shader in the fill stack into a texture, runs the pipeline into an+ * offscreen WebGPU canvas, and hands the result back to the Canvas2D+ * compositor via `drawImage`. Created lazily by `resolveShaderHost`, exactly+ * like the media decoders.+ *+ * The authored contract (see reference/jsx/shader-paint.md): the user writes+ * `@fragment fn main(@location(0) uv: vec2f) -> @location(0) vec4f` with `uv`+ * in box space, samples the media through `sampleSource(uv)` (which honors+ * the media paint's `objectFit` placement), and declares parameters as+ * `@group(1)` uniforms whose values arrive by name from the `uniforms` prop.+ */++// One device for every shader paint in the page; pipelines are per-host.+let devicePromise: Promise<GPUDevice | null> | null = null;++function sharedDevice(): Promise<GPUDevice | null> {+ devicePromise ??= (async () => {+ const adapter = await navigator.gpu?.requestAdapter();+ if (!adapter) {+ console.error("[shader-paint] WebGPU is not available; shader paints render as passthrough");+ return null;+ }+ return await adapter.requestDevice();+ })();+ return devicePromise;+}++const PRELUDE = /* wgsl */ `+struct Globals {+ resolution: vec2f,+ time: f32,+ _pad: f32,+ fitOffset: vec2f,+ fitSize: vec2f,+}++@group(0) @binding(0) var source: texture_2d<f32>;+@group(0) @binding(1) var sourceSampler: sampler;+@group(0) @binding(2) var<uniform> globals: Globals;++struct VSOut {+ @builtin(position) position: vec4f,+ @location(0) uv: vec2f,+}++@vertex+fn vs(@builtin(vertex_index) i: u32) -> VSOut {+ var corners = array<vec2f, 3>(vec2f(-1.0, -1.0), vec2f(3.0, -1.0), vec2f(-1.0, 3.0));+ var out: VSOut;+ out.position = vec4f(corners[i], 0.0, 1.0);+ out.uv = vec2f((corners[i].x + 1.0) * 0.5, 1.0 - (corners[i].y + 1.0) * 0.5);+ return out;+}++// Samples the media in box space: uv spans the parent's box, the media sits+// within it per objectFit, and letterbox areas return transparent black.+fn sampleSource(uv: vec2f) -> vec4f {+ let suv = (uv - globals.fitOffset) / globals.fitSize;+ let inside = all(suv >= vec2f(0.0)) && all(suv <= vec2f(1.0));+ let color = textureSampleLevel(source, sourceSampler, clamp(suv, vec2f(0.0), vec2f(1.0)), 0.0);+ return select(vec4f(0.0), color, inside);+}+`;++const UNIFORM_COMPONENTS: Record<string, number> = {+ f32: 1,+ vec2f: 2, "vec2<f32>": 2,+ vec3f: 3, "vec3<f32>": 3,+ vec4f: 4, "vec4<f32>": 4,+};++const UNIFORM_RE = /@group\(1\)\s*@binding\((\d+)\)\s*var\s*<\s*uniform\s*>\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*([A-Za-z0-9_]+(?:<[A-Za-z0-9_]+>)?)/g;++type UniformSlot = {+ name: string;+ binding: number;+ components: number;+ buffer: GPUBuffer;+};++type Pipeline = {+ device: GPUDevice;+ pipeline: GPURenderPipeline;+ sampler: GPUSampler;+ globals: GPUBuffer;+ slots: UniformSlot[];+ group1: GPUBindGroup;+ group0Layout: GPUBindGroupLayout;+};++export class ShaderHost {+ private canvas = new OffscreenCanvas(1, 1);+ private context: GPUCanvasContext | null = null;+ private gpu: Pipeline | null = null;+ private sourceTexture: GPUTexture | null = null;+ private group0: GPUBindGroup | null = null;+ private code = "";+ private generation = 0;+ private compilePromise: Promise<void> | null = null;+ private colorCache = new Map<string, number | null>();+ private disposed = false;++ /** Compiles when the code differs from the current pipeline's. */+ public setCode(code: string): void {+ if (this.disposed || code === this.code) return;+ this.code = code;+ const gen = ++this.generation;+ this.compilePromise = this.compile(code, gen).catch((error) => {+ if (gen === this.generation) this.gpu = null;+ console.error("[shader-paint]", error);+ });+ }++ /** Resolves once the current code has compiled (or failed). */+ public whenReady(): Promise<void> | null {+ return this.compilePromise;+ }++ /** True when a pipeline exists — callers fall back to drawing the media directly otherwise. */+ public get ready(): boolean {+ return this.gpu !== null && !this.disposed;+ }++ private async compile(code: string, gen: number): Promise<void> {+ const device = await sharedDevice();+ if (!device || this.disposed) return;++ const module = device.createShaderModule({ code: PRELUDE + code });+ const info = await module.getCompilationInfo();+ if (gen !== this.generation) return;++ const preludeLines = PRELUDE.split("\n").length;+ const errors = info.messages.filter((m) => m.type === "error");+ if (errors.length > 0) {+ this.gpu = null;+ const details = errors+ .map((m) => `${Math.max(1, m.lineNum - preludeLines + 1)}:${m.linePos} ${m.message}`)+ .join("\n");+ throw new Error(`WGSL compile failed:\n${details}`);+ }++ const group0Layout = device.createBindGroupLayout({+ entries: [+ { binding: 0, visibility: GPUShaderStage.FRAGMENT, texture: { sampleType: "float" } },+ { binding: 1, visibility: GPUShaderStage.FRAGMENT, sampler: { type: "filtering" } },+ { binding: 2, visibility: GPUShaderStage.FRAGMENT, buffer: { type: "uniform" } },+ ],+ });++ // Declared @group(1) uniforms get one small buffer each, matched to the+ // `uniforms` record by name. An explicit layout keeps declared-but-unused+ // bindings valid (an "auto" layout would strip them).+ const slots: UniformSlot[] = [];+ for (const match of code.matchAll(UNIFORM_RE)) {+ const components = UNIFORM_COMPONENTS[match[3]];+ if (components === undefined) {+ console.warn(`[shader-paint] uniform \`${match[2]}\` has unsupported type ${match[3]} (use f32/vec2f/vec3f/vec4f)`);+ continue;+ }+ slots.push({+ name: match[2],+ binding: Number(match[1]),+ components,+ buffer: device.createBuffer({+ size: components * 4,+ usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,+ }),+ });+ }++ const group1Layout = device.createBindGroupLayout({+ entries: slots.map((slot) => ({+ binding: slot.binding,+ visibility: GPUShaderStage.FRAGMENT,+ buffer: { type: "uniform" as const },+ })),+ });++ const pipeline = device.createRenderPipeline({+ layout: device.createPipelineLayout({ bindGroupLayouts: [group0Layout, group1Layout] }),+ vertex: { module, entryPoint: "vs" },+ fragment: {+ module,+ entryPoint: "main",+ targets: [{ format: navigator.gpu.getPreferredCanvasFormat() }],+ },+ });+ if (gen !== this.generation || this.disposed) return;++ this.gpu = {+ device,+ pipeline,+ sampler: device.createSampler({ magFilter: "linear", minFilter: "linear" }),+ globals: device.createBuffer({ size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST }),+ slots,+ group1: device.createBindGroup({+ layout: group1Layout,+ entries: slots.map((slot) => ({ binding: slot.binding, resource: { buffer: slot.buffer } })),+ }),+ group0Layout,+ };+ // The source texture and its bind group belong to the old pipeline's layout.+ this.sourceTexture?.destroy();+ this.sourceTexture = null;+ this.group0 = null;+ }++ private writeUniforms(uniforms: Record<string, UniformValue> | null): void {+ const gpu = this.gpu!;+ for (const slot of gpu.slots) {+ const value = uniforms?.[slot.name];+ if (value === undefined) continue;+ const data = new Float32Array(slot.components);+ if (typeof value === "number") {+ data[0] = value;+ } else if (Array.isArray(value)) {+ for (let i = 0; i < Math.min(slot.components, value.length); i++) data[i] = value[i];+ } else {+ if (!this.colorCache.has(value)) this.colorCache.set(value, parseColor(value));+ const color = this.colorCache.get(value);+ if (color == null) continue;+ data[0] = ((color >> 16) & 0xff) / 255;+ if (slot.components >= 2) data[1] = ((color >> 8) & 0xff) / 255;+ if (slot.components >= 3) data[2] = (color & 0xff) / 255;+ if (slot.components >= 4) data[3] = 1;+ }+ gpu.device.queue.writeBuffer(slot.buffer, 0, data);+ }+ }++ /**+ * Runs the shader over `source` and draws the result into the parent's+ * box. `fit` is the media's drawImage placement within the box (from+ * getScaledImageProps); `time` is the parent's local time in seconds.+ * A null `source` runs the shader procedurally: the bound texture stays+ * transparent black, so `sampleSource` returns vec4f(0) everywhere.+ * Returns false when no pipeline is ready — the caller then draws the+ * media directly instead.+ */+ public draw(+ ctx: Ctx2D,+ width: number,+ height: number,+ source: GPUCopyExternalImageSource | null,+ sourceWidth: number,+ sourceHeight: number,+ fit: [dx: number, dy: number, sw: number, sh: number],+ time: number,+ uniforms: Record<string, UniformValue> | null,+ ): boolean {+ const gpu = this.gpu;+ if (!gpu || this.disposed) return false;++ const w = Math.max(1, Math.round(width));+ const h = Math.max(1, Math.round(height));+ if (this.canvas.width !== w || this.canvas.height !== h) {+ this.canvas.width = w;+ this.canvas.height = h;+ }+ if (this.context === null) {+ this.context = this.canvas.getContext("webgpu");+ if (!this.context) return false;+ this.context.configure({+ device: gpu.device,+ format: navigator.gpu.getPreferredCanvasFormat(),+ alphaMode: "premultiplied",+ });+ }++ const sw = source ? Math.max(1, Math.round(sourceWidth)) : 1;+ const sh = source ? Math.max(1, Math.round(sourceHeight)) : 1;+ if (!this.sourceTexture || this.sourceTexture.width !== sw || this.sourceTexture.height !== sh || !this.group0) {+ this.sourceTexture?.destroy();+ this.sourceTexture = gpu.device.createTexture({+ size: [sw, sh],+ format: "rgba8unorm",+ usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.RENDER_ATTACHMENT,+ });+ this.group0 = gpu.device.createBindGroup({+ layout: gpu.group0Layout,+ entries: [+ { binding: 0, resource: this.sourceTexture.createView() },+ { binding: 1, resource: gpu.sampler },+ { binding: 2, resource: { buffer: gpu.globals } },+ ],+ });+ }++ if (source) {+ gpu.device.queue.copyExternalImageToTexture(+ { source },+ { texture: this.sourceTexture, premultipliedAlpha: true },+ [sw, sh],+ );+ }++ const [dx, dy, dw, dh] = fit;+ gpu.device.queue.writeBuffer(gpu.globals, 0, new Float32Array([+ w, h, time, 0,+ dx / w, dy / h, dw / w, dh / h,+ ]));+ this.writeUniforms(uniforms);++ const encoder = gpu.device.createCommandEncoder();+ const pass = encoder.beginRenderPass({+ colorAttachments: [{+ view: this.context.getCurrentTexture().createView(),+ clearValue: { r: 0, g: 0, b: 0, a: 0 },+ loadOp: "clear",+ storeOp: "store",+ }],+ });+ pass.setPipeline(gpu.pipeline);+ pass.setBindGroup(0, this.group0);+ pass.setBindGroup(1, gpu.group1);+ pass.draw(3);+ pass.end();+ gpu.device.queue.submit([encoder.finish()]);++ ctx.drawImage(this.canvas, 0, 0, width, height);+ return true;+ }++ public dispose(): void {+ if (this.disposed) return;+ this.disposed = true;+ this.generation++;+ this.sourceTexture?.destroy();+ this.sourceTexture = null;+ if (this.gpu) {+ this.gpu.globals.destroy();+ for (const slot of this.gpu.slots) slot.buffer.destroy();+ this.gpu = null;+ }+ this.context?.unconfigure();+ this.canvas.width = 0;+ this.canvas.height = 0;+ }+}++/**+ * The shader paint's host, created lazily like the media decoders and kept in+ * sync with the persisted `Shader.code` component. Returns null until the+ * paint carries code.+ */+export function resolveShaderHost(_world: World, entity: Entity): ShaderHost | null {+ const code = entity.get(Shader)?.code;+ if (!code) return null;++ let host = entity.get(ShaderHostHandle) ?? null;+ if (!host) {+ host = new ShaderHost();+ entity.add(ShaderHostHandle);+ entity.set(ShaderHostHandle, host);+ }+ host.setCode(code);+ return host;+}diff --git a/packages/runtime/src/media/surface.ts b/packages/runtime/src/media/surface.tsnew file mode 100644index 00000000..c4d5a803--- /dev/null+++ b/packages/runtime/src/media/surface.ts@@ -0,0 +1,39 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;++/**+ * Backing store for one surface paint: a detached canvas handed to the+ * mount's `ref` callback, which draws into it with any context type (2d,+ * webgl, webgpu). The engine samples the bitmap every frame and stretches it+ * into the parent geometry's box. After the initial box-sized allocation the+ * bitmap belongs to user code — external renderers (e.g. three.js via+ * `renderer.setSize`) resize it through their own APIs, so the engine never+ * touches it again.+ */+export class SurfaceHost {+ public readonly canvas = document.createElement('canvas');+ private disposed = false;++ /** Initial bitmap allocation — called once at materialization, before the ref runs. */+ public setSize(width: number, height: number): void {+ this.canvas.width = Math.max(1, Math.round(width));+ this.canvas.height = Math.max(1, Math.round(height));+ }++ public draw(ctx: Ctx2D, width: number, height: number): void {+ if (this.disposed || this.canvas.width === 0 || this.canvas.height === 0) return;+ ctx.drawImage(this.canvas, 0, 0, width, height);+ }++ public dispose(): void {+ if (this.disposed) return;+ this.disposed = true;+ // Release the backing store; context teardown (e.g. a WebGL renderer)+ // is the mount's own onCleanup concern.+ this.canvas.width = 0;+ this.canvas.height = 0;+ }+}diff --git a/packages/runtime/src/media/time-stretcher.ts b/packages/runtime/src/media/time-stretcher.tsnew file mode 100644index 00000000..4af25cf5--- /dev/null+++ b/packages/runtime/src/media/time-stretcher.ts@@ -0,0 +1,260 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */+++/**+ * Utility class for performing time stretching on a multi-channel audio signal. The input audio will be stretched by+ * a configurable factor without changing its pitch.+ *+ * Internally, this uses a WSOLA-like algorithm.+ */+export class TimeStretcher {+ factor: number; // This value can be changed at runtime++ private numberOfChannels: number;+ private synthesisHopSize: number;+ private windowSize: number;+ private overlapSize: number;+ private tolerance: number;+ private buffers: Float32Array[];+ private bufferEndIndex: number;+ private outputBuffers: Float32Array[];+ private nextOutputBufferShiftAmount: number;+ private outputBufferLength: number;+ private hasDoneOutput: boolean;+ private finalized: boolean;+ private blendValues: number[];++ public constructor(numberOfChannels: number, sampleRate: number, factor: number) {+ this.numberOfChannels = numberOfChannels;+ this.synthesisHopSize = Math.floor((512 * sampleRate) / 48000);+ this.windowSize = 2 * this.synthesisHopSize;+ this.overlapSize = this.synthesisHopSize;+ this.tolerance = this.synthesisHopSize;+ this.buffers = [];+ this.bufferEndIndex = this.tolerance;+ this.factor = factor;+ this.outputBuffers = [];+ this.nextOutputBufferShiftAmount = 0;+ this.outputBufferLength = 2 ** 16;+ this.hasDoneOutput = false;+ this.finalized = false;++ for (let i = 0; i < numberOfChannels; i++) {+ this.buffers.push(new Float32Array(2 ** 16));+ this.outputBuffers.push(new Float32Array(this.outputBufferLength));+ }++ this.blendValues = [];+ for (let i = 0; i < this.overlapSize; i++) {+ this.blendValues[i] = 0.5 * (1 - Math.cos((Math.PI * i) / this.overlapSize));+ }+ }++ private ensureOutputBufferLength(requiredLength: number): void {+ if (requiredLength > this.outputBufferLength) {+ this.outputBufferLength = requiredLength;+ for (let i = 0; i < this.numberOfChannels; i++) {+ const largerBuffer = new Float32Array(this.outputBufferLength);+ largerBuffer.set(this.outputBuffers[i], 0);+ this.outputBuffers[i] = largerBuffer;+ }+ }+ }++ private clearOutputBuffers(): void {+ if (this.nextOutputBufferShiftAmount > 0) {+ for (let i = 0; i < this.numberOfChannels; i++) {+ const buffer = this.outputBuffers[i];+ buffer.set(buffer.subarray(this.nextOutputBufferShiftAmount), 0);+ }++ this.nextOutputBufferShiftAmount = 0;+ }+ }++ private synthesizeSegment(+ i: number,+ windowSize: number,+ inputStartPos: number,+ maxPositiveOffset: number = this.tolerance+ ): void {+ const crossCorrelateAllChannels = (k: number): number => {+ let dot = 0;+ let normOldTotal = 0;+ let normNewTotal = 0;++ for (let chan = 0; chan < this.numberOfChannels; chan++) {+ const inputBuffer = this.buffers[chan];+ const outputBuffer = this.outputBuffers[chan];++ for (let j = 0; j < this.overlapSize; j++) {+ const oldValue = outputBuffer[i + j];+ const newValue = inputBuffer[inputStartPos + k + j];++ dot += oldValue * newValue;+ normOldTotal += oldValue ** 2;+ normNewTotal += newValue ** 2;+ }+ }++ const ncc = dot / (Math.sqrt(normOldTotal * normNewTotal) || 1e-10);+ return ncc;+ };++ let bestCorr = -Infinity;+ let bestOffset = 0;++ if (!this.hasDoneOutput || this.factor === 1) {+ bestOffset = 0;+ } else {+ let k = -this.tolerance;+ bestCorr = crossCorrelateAllChannels(k);+ bestOffset = k;++ const minStepSize = 1;+ const maxStepSize = 16;+ let prevCorr = bestCorr;++ while (k < maxPositiveOffset) {+ const nextK = Math.min(k + 1, maxPositiveOffset);+ const corr = crossCorrelateAllChannels(nextK);++ const gradient = Math.abs(corr - prevCorr);+ const adaptiveStep = Math.max(+ minStepSize,+ Math.min(maxStepSize, Math.floor(maxStepSize * Math.exp(-gradient * 3)))+ );++ if (corr > bestCorr) {+ bestCorr = corr;+ bestOffset = nextK;+ }++ prevCorr = corr;+ k += adaptiveStep;+ }++ // Fine-tuning phase+ const fineRange = 8;+ for (k = bestOffset - fineRange; k <= bestOffset + fineRange; k++) {+ if (k >= -this.tolerance && k <= maxPositiveOffset) {+ const corr = crossCorrelateAllChannels(k);+ if (corr > bestCorr) {+ bestCorr = corr;+ bestOffset = k;+ }+ }+ }+ }++ for (let chan = 0; chan < this.numberOfChannels; chan++) {+ const inputBuffer = this.buffers[chan];+ const outputBuffer = this.outputBuffers[chan];++ for (let j = 0; j < windowSize; j++) {+ const blendValue = j < this.overlapSize ? this.blendValues[j] : 1;+ outputBuffer[i + j] *= 1 - blendValue;+ outputBuffer[i + j] += blendValue * inputBuffer[inputStartPos + bestOffset + j];+ }+ }++ this.hasDoneOutput = true;+ }++ public append(newBuffers: Float32Array[]): Float32Array[] | null {+ if (this.finalized) {+ throw new Error('Cannot append more buffers after calling finalize.');+ }++ // Fast path: when factor is 1, pass through directly without any processing+ if (this.factor === 1) {+ this.hasDoneOutput = true;+ return newBuffers;+ }++ for (let i = 0; i < this.numberOfChannels; i++) {+ const newBuffer = newBuffers[i];+ const requiredLength = this.bufferEndIndex + newBuffer.length;+ let currentLength = this.buffers[i].length;++ while (currentLength < requiredLength) {+ currentLength *= 2;+ }++ if (currentLength !== this.buffers[i].length) {+ const largerBuffer = new Float32Array(currentLength);+ largerBuffer.set(this.buffers[i], 0);+ this.buffers[i] = largerBuffer;+ }++ this.buffers[i].set(newBuffer, this.bufferEndIndex);+ }++ this.bufferEndIndex += newBuffers[0].length;++ return this.process();+ }++ private process(): Float32Array[] | null {+ let synthesisLength = 0;+ // eslint-disable-next-line no-constant-condition+ for (synthesisLength; true; synthesisLength += this.synthesisHopSize) {+ const inputShiftAmount = Math.floor(synthesisLength / this.factor);++ if (inputShiftAmount + this.windowSize + 2 * this.tolerance > this.bufferEndIndex) {+ synthesisLength -= this.synthesisHopSize;+ break;+ }+ }++ if (synthesisLength <= 0) {+ return null;+ }++ const inputShiftAmount = Math.floor(synthesisLength / this.factor);+ const outputLength = synthesisLength - this.synthesisHopSize + this.windowSize;++ this.clearOutputBuffers();+ this.ensureOutputBufferLength(outputLength);++ for (let i = 0; i < synthesisLength; i += this.synthesisHopSize) {+ const inputStartPos = this.tolerance + Math.floor(i / this.factor);+ this.synthesizeSegment(i, this.windowSize, inputStartPos);+ }++ // Shift all input buffers+ for (let chan = 0; chan < this.numberOfChannels; chan++) {+ const inputBuffer = this.buffers[chan];+ inputBuffer.set(inputBuffer.subarray(inputShiftAmount, this.bufferEndIndex), 0);+ }++ this.bufferEndIndex -= inputShiftAmount;+ this.nextOutputBufferShiftAmount = synthesisLength;++ return this.outputBuffers.map(x => x.subarray(0, synthesisLength));+ }++ public finalize(): Float32Array[] | null {+ this.finalized = true;++ // Fast path: when factor is 1, nothing is buffered+ if (this.factor === 1) {+ return null;+ }++ if (this.bufferEndIndex <= this.tolerance) {+ return null;+ }++ this.clearOutputBuffers();++ const synthesisLength = this.bufferEndIndex - this.tolerance;+ this.ensureOutputBufferLength(synthesisLength);++ this.synthesizeSegment(0, synthesisLength, this.tolerance, 0);++ return this.outputBuffers.map(x => x.subarray(0, synthesisLength));+ }+}diff --git a/packages/runtime/src/media/video.ts b/packages/runtime/src/media/video.tsnew file mode 100644index 00000000..b8aa86bc--- /dev/null+++ b/packages/runtime/src/media/video.ts@@ -0,0 +1,731 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { BlobSource, ALL_FORMATS, Input, InputVideoTrack, EncodedPacketSink, EncodedPacket, CanvasSink, type WrappedCanvas } from 'mediabunny';++import { AssetId, VideoDecoderHandle, Assets, Mode } from '../traits';+import { assert } from '../utils/assert';+import { getAssetFile } from '../actions/assets';+import { FrameCache } from './frame-cache';+import { getKeyframeIndex } from './keyframe-index';++import type { Entity, World } from 'koota';+import type { KeyframeIndex } from './keyframe-index';+import type { VideoAsset } from '../assets/types';+++export type VideoBufferMode = 'discarded' | 'idle' | 'alive';++/**+ * Inactivity window after which an `alive` buffer automatically drops to+ * `idle`, freeing its decoder and frame cache while keeping the display canvas.+ */+const IDLE_TIMEOUT_MS = 10_000;++/**+ * Max extra frames we are willing to decode forward from the live cursor in+ * order to avoid a keyframe lookup + decoder reseed. Higher = stronger bias.+ */+const FORWARD_BIAS_FRAMES = 24;++/**+ * Frames used to drain the decoder in case of a backward seek to+ * ensure the target frame is surfaced.+ */+const DRAIN_FRAMES = 8;++/**+ * How far the preview may drift from the requested frame. A neighbour this close+ * beats holding the last drawn frame while the exact one is still decoding.+ */+const DISPLAY_TOLERANCE_FRAMES = 4;++/**+ * Total pixel budget of the preview frame cache+ */+const CACHE_PIXEL_BUDGET = 768 * 432 * 81; // 81 tiles at 768x432++/**+ * Per-tile pixel cap (~720p) — enough detail for the preview canvas.+ */+const MAX_TILE_PIXELS = 1280 * 720;++/**+ * Two seeks arriving closer together than this are treated as one continuous drag.+ */+const SCRUB_EVENT_WINDOW_MS = 250;++/**+ * Quiet period after the last scrub seek before the exact frame is resolved.+ */+const SCRUB_SETTLE_MS = 120;++const MIN_CACHE_COUNT = 30;+const MAX_CACHE_COUNT = 81;++export class VideoBuffer {+ public errored = false;+ public asset: VideoAsset;+ public firstPacketTimestamp = 0;+ public packetSink: EncodedPacketSink | null = null;+ public mode: VideoBufferMode = 'alive';+ public initialized: Promise<void>;++ public readonly cache: FrameCache;+ public readonly queue = new VideoDecoderQueue(this.frameCallback.bind(this));++ // user facing display canvas+ public readonly canvas = new OffscreenCanvas(0, 0);+ public readonly ctx = this.canvas.getContext('2d')!;++ private currentFrame: number = -1;+ private isDirty: boolean = true;+ private keyframes: KeyframeIndex | null = null;++ /**+ * Frame the preview is aiming to show. Tracks `currentFrame` except while scrubbing,+ * where it points at the covering keyframe instead of the requested frame.+ */+ private displayFrame: number = -1;++ /**+ * Keyframes a scrub is waiting on. The preview keeps showing the last one that landed+ * until the next arrives, so a drag never blanks out for the length of a decode.+ * Holds several at once: decoder output trails submission by a handful of packets.+ */+ private readonly pendingScrub = new Set<number>();++ private lastFrameIndex: number = 0;+ private seekGeneration = 0;+ private seekLock: Promise<void> = Promise.resolve();+ private iterator: AsyncGenerator<EncodedPacket, void, unknown> | null = null;+ private idleTimer: ReturnType<typeof setTimeout> | null = null;+ private settleTimer: ReturnType<typeof setTimeout> | null = null;+ private lastSeekAt: number = -Infinity;++ public constructor(asset: VideoAsset) {+ this.asset = asset;++ const pixels = Math.min(asset.width * asset.height, MAX_TILE_PIXELS);+ const count = Math.min(MAX_CACHE_COUNT, Math.max(MIN_CACHE_COUNT, Math.floor(CACHE_PIXEL_BUDGET / pixels)));+ this.cache = new FrameCache({ pixels, count });++ this.initialized = this.initialize();+ }++ private async initialize() {+ try {+ const videoTrack = await getVideoTrack(this.asset);++ assert(videoTrack, 'Video track not found');++ this.keyframes = getKeyframeIndex(this.asset.id, videoTrack);++ await this.queue.init(videoTrack);++ this.cache.rotation = videoTrack.rotation;+ this.packetSink = new EncodedPacketSink(videoTrack);+ this.firstPacketTimestamp = Math.max(0, await videoTrack.getFirstTimestamp() ?? 0);+ this.lastFrameIndex = Math.max(0, Math.round(this.asset.duration * this.asset.frameRate) - 1);+ this.isDirty = true;+ } catch (e) {+ console.error('Error initializing video decoder', e);+ this.errored = true;+ }+ }++ private frameCallback(frame: VideoFrame) {+ const timestampSeconds = frame.timestamp / 1e6;+ const frameIndex = this.secondsToFrames(timestampSeconds);+ this.cache.insert(frame, frameIndex);+ this.isDirty = true;++ // A scrub keyframe takes over the preview the moment it lands, and only then:+ // matching on the index keeps frames from an abandoned fill out of the display.+ if (this.pendingScrub.delete(frameIndex)) {+ this.displayFrame = frameIndex;+ }+ }++ public seekTo(frame: number, frameRate: number): undefined {+ const targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);++ const isEmpty = !this.packetSink;+ const isCurrentFrame = targetFrame === this.currentFrame;+ const isCached = this.cache.has(targetFrame) && isCurrentFrame;+ const isIdle = this.mode === 'idle' && isCurrentFrame;+ const errored = this.errored;++ if (isEmpty || isCurrentFrame || isCached || isIdle || errored) {+ return;+ }++ const previousFrame = this.currentFrame;+ const consecutive = performance.now() - this.lastSeekAt < SCRUB_EVENT_WINDOW_MS;+ this.lastSeekAt = performance.now();++ this.currentFrame = targetFrame;+ this.isDirty = true;++ this.touch();++ // A run of large jumps is a drag, not a playback step. Walking each GOP to the+ // exact frame costs more than the gap between pointer events, so every walk gets+ // cancelled by the next one and nothing ever paints.+ const jumped = Math.abs(targetFrame - previousFrame) > FORWARD_BIAS_FRAMES;++ if (consecutive && jumped && this.scrubTo(targetFrame)) {+ return;+ }++ this.exactSeekTo(targetFrame, previousFrame);+ }++ /**+ * Paints the keyframe covering `targetFrame` rather than the frame itself: one packet,+ * no walk through the GOP. Returns false when the keyframe index cannot place the+ * target yet, leaving the caller to fall back to an exact seek.+ */+ private scrubTo(targetFrame: number): boolean {+ const keyTimestamp = this.keyframes?.floor(this.framesToSeconds(targetFrame)) ?? null;++ if (keyTimestamp === null) {+ return false;+ }++ const keyFrame = this.secondsToFrames(keyTimestamp);+ this.scheduleSettle();++ // Supersedes any fill still running for a position we have already left.+ const generation = ++this.seekGeneration;++ // Consecutive positions inside one GOP resolve to a keyframe we already hold.+ if (this.cache.has(keyFrame)) {+ this.displayFrame = keyFrame;+ return true;+ }++ this.pendingScrub.add(keyFrame);++ this.seekLock = this.seekLock+ .then(() => this.decodeKeyframe(keyTimestamp, generation))+ .catch(() => { });++ return true;+ }++ private async decodeKeyframe(keyTimestamp: number, generation: number): Promise<void> {+ if (generation !== this.seekGeneration) return;++ const keyPacket = await this.packetSink?.getKeyPacket(keyTimestamp);+ if (!keyPacket || generation !== this.seekGeneration) return;++ // Seed the iterator here as well, so the settle pass continues forward from this+ // keyframe instead of reseeding the decoder a second time.+ await this.iterator?.return();+ const iterator = this.packetSink?.packets(keyPacket) ?? null;+ this.iterator = iterator;+ if (!iterator) return;++ if (!this.queue.isAlive) {+ this.queue.reseed();+ }++ const { value: packet } = await iterator.next();+ if (packet) {+ await this.queue.decode(packet);+ }+ }++ /**+ * Re-runs the seek for real once the drag stops.+ */+ private scheduleSettle() {+ if (this.settleTimer !== null) {+ clearTimeout(this.settleTimer);+ }++ this.settleTimer = setTimeout(() => {+ this.settleTimer = null;+ if (this.mode !== 'alive' || this.errored) return;+ this.exactSeekTo(this.currentFrame, this.currentFrame);+ }, SCRUB_SETTLE_MS);+ }++ private exactSeekTo(targetFrame: number, previousFrame: number) {+ this.displayFrame = targetFrame;+ this.pendingScrub.clear();++ const forward = this.isBlockedFrame(targetFrame)+ ? targetFrame >= previousFrame+ : true;++ const generation = ++this.seekGeneration;++ const [left, right] = this.computeWindow(targetFrame, forward);+ this.cache.leftFrameIndex = left;+ this.cache.rightFrameIndex = right;++ let seed = targetFrame;+ let run: Promise<void>+ if (forward) {+ while (this.isBlockedFrame(seed) && seed <= this.cache.rightFrameIndex) {+ seed++;+ }++ run = this.seekLock.then(() => this.fillCache([seed, this.cache.rightFrameIndex], generation));+ } else {+ while (this.cache.has(seed) && seed >= this.cache.leftFrameIndex) {+ seed--;+ }++ // TODO: Find a better solution for DRAIN_FRAMES+ run = this.seekLock.then(() => this.fillCache([this.cache.leftFrameIndex, seed + DRAIN_FRAMES], generation));+ }++ // Run after the previous seek finishes — never concurrently.+ this.seekLock = run.catch(() => { });+ }++ private framesToSeconds(frames: number) {+ return Math.max(this.firstPacketTimestamp, (frames / this.asset.frameRate) + this.firstPacketTimestamp);+ }++ private secondsToFrames(seconds: number) {+ return Math.max(0, Math.round((seconds - this.firstPacketTimestamp) * this.asset.frameRate));+ }++ /**+ * Whether `frame` is already taken care of — decoded into the cache, or still in-flight.+ */+ private isBlockedFrame(frame: number) {+ if (this.cache.has(frame)) return true;++ for (const micros of this.queue.inFlight) {+ if (this.secondsToFrames(micros / 1e6) === frame) {+ return true;+ }+ }++ return false;+ }++ private computeWindow(targetFrame: number, forward: boolean): [number, number] {+ const span = this.cache.config.count - 2;++ // Whole video fits in the cache — keep all of it.+ if (this.lastFrameIndex <= span) {+ return [0, this.lastFrameIndex];+ }++ const ahead = Math.round((span * 2) / 3);+ const behind = span - ahead;++ let left = targetFrame - (forward ? behind : ahead);+ let right = targetFrame + (forward ? ahead : behind);++ // Push budget that falls outside the boundaries onto the other side.+ if (left < 0) {+ right -= left;+ left = 0;+ }+ if (right > this.lastFrameIndex) {+ left -= right - this.lastFrameIndex;+ right = this.lastFrameIndex;+ }++ return [Math.max(0, left), Math.min(this.lastFrameIndex, right)];+ }++ private async fillCache(range: [number, number], generation: number): Promise<void> {+ if (generation !== this.seekGeneration || Math.abs(range[1] - range[0]) <= 3) return;+ // Note: only decode for three or more frames at a time to avoid unnecessary reseeding++ const fromSecs = this.framesToSeconds(range[0]);+ const untilSecs = this.framesToSeconds(range[1]);+ const cursor = this.queue.lastSubmitted;+ const live = !!(this.iterator && this.queue.isAlive && cursor);++ let reuse = live && cursor!.timestamp < untilSecs;+ let keyTimestamp = this.keyframes?.floor(fromSecs) ?? null;++ // If the cursor lags far behind the range start, skip forward to the nearest+ // keyframe rather than decoding through the whole gap.+ const biasSecs = FORWARD_BIAS_FRAMES / this.asset.frameRate;+ if (reuse && cursor!.timestamp < fromSecs - biasSecs) {+ keyTimestamp ??= (await this.packetSink?.getKeyPacket(fromSecs))?.timestamp ?? null;+ if (keyTimestamp !== null && keyTimestamp - cursor!.timestamp > biasSecs) {+ reuse = false; // jumping to the keyframe+ }+ }++ // we cant reuse the iterator, so we need to reseed the decoder+ // and get create a new iterator+ if (!reuse) {+ const keyPacket = (await this.packetSink?.getKeyPacket(keyTimestamp ?? fromSecs)) ?? null;+ if (!keyPacket) return;+ await this.iterator?.return();+ this.iterator = this.packetSink?.packets(keyPacket) ?? null;+ }++ if (!this.queue.isAlive) {+ this.queue.reseed();+ }++ if (generation !== this.seekGeneration || !this.iterator) return;++ while (true) {+ const { value: packet, done } = await this.iterator.next();+ if (done || !packet) break;++ await this.queue.decode(packet);++ if (generation !== this.seekGeneration || packet.timestamp >= untilSecs) break;+ }+ }++ public toBitmap() {+ if (this.isDirty && this.displayFrame >= 0) {++ const nearest = this.cache.findNearest(this.displayFrame, DISPLAY_TOLERANCE_FRAMES);+ const tile = nearest === undefined ? undefined : this.cache.findTile(nearest);+ const ctx = this.ctx;++ if (tile && (this.canvas.width !== tile.width || this.canvas.height !== tile.height)) {+ this.canvas.width = tile.width;+ this.canvas.height = tile.height;+ this.ctx.imageSmoothingEnabled = false;+ }++ if (tile) {+ ctx.drawImage(+ this.cache.atlas,+ tile.x,+ tile.y,+ tile.width,+ tile.height,+ 0,+ 0,+ tile.width,+ tile.height,+ );+ this.isDirty = false;+ }+ }++ if (this.canvas.width === 0 || this.canvas.height === 0) {+ return null;+ }++ return this.canvas;+ }++ private touch() {+ this.mode = 'alive';+ if (this.idleTimer !== null) {+ clearTimeout(this.idleTimer);+ }+ this.idleTimer = setTimeout(() => this.idle(), IDLE_TIMEOUT_MS);+ }++ public idle() {+ if (this.mode !== 'alive') return;+ this.mode = 'idle';++ if (this.idleTimer !== null) {+ clearTimeout(this.idleTimer);+ this.idleTimer = null;+ }++ if (this.settleTimer !== null) {+ clearTimeout(this.settleTimer);+ this.settleTimer = null;+ }++ this.pendingScrub.clear();+ this.cache.dispose();+ this.queue.dispose();+ this.iterator?.return();+ this.iterator = null;+ }++ public dispose() {+ if (this.mode === 'discarded') return;+ this.mode = 'discarded';++ if (this.idleTimer !== null) {+ clearTimeout(this.idleTimer);+ this.idleTimer = null;+ }++ if (this.settleTimer !== null) {+ clearTimeout(this.settleTimer);+ this.settleTimer = null;+ }++ this.pendingScrub.clear();+ this.cache.dispose();+ this.queue.dispose();+ this.iterator?.return();+ this.iterator = null;++ // Release the display canvas backing store.+ this.canvas.width = 0;+ this.canvas.height = 0;+ }+}++class VideoDecoderQueue {+ private config: VideoDecoderConfig | null = null;+ private decoder: VideoDecoder | null = null;+ private resolver: ReturnType<typeof Promise.withResolvers> | null = null;+ private callback: (frame: VideoFrame) => void;+ public lastSubmitted: EncodedPacket | null = null;+ public readonly inFlight = new Set<number>();++ public constructor(callback: (frame: VideoFrame) => void) {+ this.callback = callback;+ }++ public get isAlive() {+ return this.decoder?.state === 'configured';+ }++ /**+ * Synchronously discard all in-flight work and re-arm the decoder for a fresh keyframe+ */+ public reseed() {+ if (!this.decoder) return;+ this.decoder.reset();+ assert(this.config, 'Decoder config not available');+ this.decoder.configure(this.config);+ this.resolver?.resolve(null);+ this.resolver = null;+ this.lastSubmitted = null;+ this.inFlight.clear();+ }++ private handleOutput(frame: VideoFrame) {+ this.resolver?.resolve(null);+ this.inFlight.delete(frame.timestamp);++ for (const micros of this.inFlight) {+ if (micros < frame.timestamp) {+ this.inFlight.delete(micros);+ }+ }++ this.callback(frame);+ frame.close();+ }++ private handleDequeue() {+ this.resolver?.resolve(null);+ this.resolver = null;+ };++ private handleError(e?: DOMException) {+ console.error(e?.message);+ this.dispose();+ }++ public async init(track: InputVideoTrack) {+ this.config = await track.getDecoderConfig();+ assert(this.config, 'Failed to get decoder config from track');+ const support = await VideoDecoder.isConfigSupported(this.config);+ assert(support.supported, 'Decoder config not supported');+ }++ private ensureDecoder(packet: EncodedPacket) {+ if (this.decoder || packet.type === 'delta') return;++ assert(this.config, 'Decoder config not available');++ this.decoder = new VideoDecoder({+ error: this.handleError.bind(this),+ output: this.handleOutput.bind(this),+ });++ this.decoder.addEventListener('dequeue', this.handleDequeue.bind(this));+ this.decoder.configure(this.config);+ }++ public async decode(packet: EncodedPacket) {+ this.ensureDecoder(packet);++ if (this.decoder?.state !== 'configured') {+ this.lastSubmitted = null;+ this.resolver?.resolve(null);+ this.resolver = null;+ this.inFlight.clear();+ return;+ }++ if (this.decoder.decodeQueueSize > 2) {+ this.resolver = Promise.withResolvers();+ }++ this.decoder.decode(packet.toEncodedVideoChunk());+ this.lastSubmitted = packet;+ this.inFlight.add(packet.microsecondTimestamp);++ await this.resolver?.promise;+ }++ public dispose() {+ try {+ this.decoder?.close();+ } catch { /* ignore */ }+ this.resolver?.resolve(null);+ this.decoder = null;+ this.lastSubmitted = null;+ this.resolver = null;+ this.inFlight.clear();+ }+}++/**+ * Dedicated, full-resolution video decoder for export.+ */+export class VideoExporter {+ public errored = false;+ public asset: VideoAsset;+ public initialized: Promise<void>;++ private input: Input | null = null;+ private canvasSink: CanvasSink | null = null;+ private iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null = null;+ private currentCanvas: WrappedCanvas | null = null;+ private firstTimestamp: number = 0;++ public constructor(asset: VideoAsset) {+ this.asset = asset;+ this.initialized = this.initialize();+ }++ private async initialize() {+ try {+ const blob = await getAssetFile(this.asset);+ this.input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ const track = await this.input.getPrimaryVideoTrack();+ assert(track, 'Video track not found');+ // See VideoBuffer.initialize: clamp so an edit-list head trim (negative first+ // timestamp) doesn't offset every exported frame relative to the audio.+ this.firstTimestamp = Math.max(0, await track.getFirstTimestamp() ?? 0);+ this.canvasSink = new CanvasSink(track, { poolSize: 2 });+ } catch (e) {+ console.error('Error initializing video exporter', e);+ this.errored = true;+ }+ }++ public async seekTo(frame: number, frameRate: number): Promise<void> {+ await this.initialized;++ if (this.errored || !this.canvasSink) return;++ const targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);+ const lastFrame = this.currentCanvas+ ? this.secondsToFrames(this.currentCanvas.timestamp)+ : -1;++ if (targetFrame === lastFrame) return;++ if (!this.iterator || targetFrame < lastFrame) {+ this.iterator = this.canvasSink?.canvases(this.framesToSeconds(targetFrame)) ?? null;+ }+ if (!this.iterator) return;++ // Walk forward while the lookahead frame still starts at/before target.+ while (true) {+ const { value, done } = await this.iterator.next();+ if (done || !value) break;++ this.currentCanvas = value;++ if (this.secondsToFrames(value.timestamp) >= targetFrame) {+ break;+ }+ }+ }++ private framesToSeconds(frames: number) {+ return Math.max(this.firstTimestamp, (frames / this.asset.frameRate) + this.firstTimestamp);+ }++ private secondsToFrames(seconds: number) {+ return Math.max(0, Math.round((seconds - this.firstTimestamp) * this.asset.frameRate));+ }++ public toBitmap(): HTMLCanvasElement | OffscreenCanvas | null {+ if (!this.currentCanvas) {+ return null;+ }++ return this.currentCanvas.canvas;+ }++ public idle(): void { }++ public dispose(): void {+ this.iterator?.return();+ this.iterator = null;+ this.input?.dispose();+ }+}++export type VideoDecoderInstance = VideoBuffer | VideoExporter;++const videoTrackCache = new Map<string, Promise<InputVideoTrack | null>>();++export function clearVideoTrackCache() {+ videoTrackCache.clear();+}++export function getVideoTrack(source: VideoAsset) {+ let promise = videoTrackCache.get(source.id);+ if (promise) {+ return promise;+ }++ promise = (async () => {+ try {+ const blob = await getAssetFile(source);+ const input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(blob)+ });+ return await input.getPrimaryVideoTrack();+ } catch {+ videoTrackCache.delete(source.id);+ return null;+ }+ })();++ videoTrackCache.set(source.id, promise);+ return promise;+}++export function resolveVideoDecoder(world: World, entity: Entity): VideoDecoderInstance | null {+ const assetId = entity.get(AssetId)?.value;+ if (!assetId) return null;++ const existing = entity.get(VideoDecoderHandle);+ if (existing && existing.asset.id === assetId) {+ return existing;+ }++ // Asset changed — dispose old decoder and create a new one.+ existing?.dispose();++ const asset = world.get(Assets)?.get(assetId);+ if (!asset || asset.type !== 'VIDEO') return null;++ const decoder = world.get(Mode)?.value === 'realtime'+ ? new VideoBuffer(asset)+ : new VideoExporter(asset);+ entity.add(VideoDecoderHandle);+ entity.set(VideoDecoderHandle, decoder);+ return decoder;+}diff --git a/packages/runtime/src/systems/gradients.ts b/packages/runtime/src/systems/gradients.tsnew file mode 100644index 00000000..9a2a0e5f--- /dev/null+++ b/packages/runtime/src/systems/gradients.ts@@ -0,0 +1,130 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Canvas gradient construction from paint sub-entities (was part of+// systems/render.ts; split out because text rendering needs it before the+// render system moves in).++import { Not } from 'koota';++import { store } from '../world/store';+import { ChildOf, Deleted, ColorStop, Position, Computed } from '../traits';+import { colorToHex } from '../utils/color';++import type { Entity, World } from 'koota';++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;++function addStopsTo(world: World, fill: Entity, gradient: CanvasGradient): void {+ const computed = store(world, Computed);+ const stops = [...world.query(ColorStop, ChildOf(fill), Not(Deleted))]+ .map(stop => {+ const raw = computed.stopOffset[stop.id()]!;+ return {+ offset: raw <= 1 ? Math.max(0, raw) : raw % 1,+ color: computed.stopColor[stop.id()]!,+ opacity: computed.stopOpacity[stop.id()]!,+ };+ })+ .sort((a, b) => a.offset - b.offset);++ for (const { offset, color, opacity } of stops) {+ gradient.addColorStop(offset, toGradientColor(color, opacity));+ }+}++function toGradientColor(color: number, opacity: number): string {+ if (opacity >= 1) return colorToHex(color);++ const r = (color >> 16) & 0xFF;+ const g = (color >> 8) & 0xFF;+ const b = color & 0xFF;+ return `rgba(${r},${g},${b},${Math.max(0, Math.min(1, opacity))})`;+}++/** Create a canvas linear gradient from a gradient paint sub-entity. */+export function createLinearGradient(+ world: World,+ fill: Entity,+ ctx: Ctx2D,+ w: number,+ h: number,+): CanvasGradient {+ const computed = store(world, Computed);+ const fid = fill.id();++ // (px, py) is the gradient center in normalized shape-local space. Fills+ // without a Position trait (the default for stock gradient fills) get+ // (0.5, 0.5) so the gradient spans the shape. With Position present, the+ // motion-system-sampled Computed values are used (keyframeable).+ const hasPos = fill.has(Position);+ const px = hasPos ? computed.positionX[fid]! : 0.5;+ const py = hasPos ? computed.positionY[fid]! : 0.5;+ const sx = computed.scaleX[fid]!;+ const sy = computed.scaleY[fid]!;+ const angle = (computed.rotation[fid]! * Math.PI) / 180;+ const cos = Math.cos(angle);+ const sin = Math.sin(angle);++ // Transform normalized start (0, 0.5) and end (1, 0.5) points+ const startLocalX = (0 - 0.5) * sx;+ const startLocalY = (0.5 - 0.5) * sy;+ const x0 = (px + startLocalX * cos - startLocalY * sin) * w;+ const y0 = (py + startLocalX * sin + startLocalY * cos) * h;++ const endLocalX = (1 - 0.5) * sx;+ const endLocalY = (0.5 - 0.5) * sy;+ const x1 = (px + endLocalX * cos - endLocalY * sin) * w;+ const y1 = (py + endLocalX * sin + endLocalY * cos) * h;++ const gradient = ctx.createLinearGradient(x0, y0, x1, y1);++ addStopsTo(world, fill, gradient);+ return gradient;+}++/** Create a canvas radial gradient from a gradient paint sub-entity. */+export function createRadialGradient(+ world: World,+ fill: Entity,+ ctx: Ctx2D,+ w: number,+ h: number,+): CanvasGradient {+ const computed = store(world, Computed);+ const fid = fill.id();++ const hasPos = fill.has(Position);+ const px = hasPos ? computed.positionX[fid]! : 0.5;+ const py = hasPos ? computed.positionY[fid]! : 0.5;+ const sx = computed.scaleX[fid]!;+ const sy = computed.scaleY[fid]!;+ const angle = (computed.rotation[fid]! * Math.PI) / 180;+ const cos = Math.cos(angle);+ const sin = Math.sin(angle);++ // Center point (0.5, 0.5) in normalized space+ const cx = px * w;+ const cy = py * h;++ // Edge points to determine radius+ const edgeXLocalX = (1 - 0.5) * sx;+ const edgeXLocalY = (0.5 - 0.5) * sy;+ const edgeXAbsX = (px + edgeXLocalX * cos - edgeXLocalY * sin) * w;+ const edgeXAbsY = (py + edgeXLocalX * sin + edgeXLocalY * cos) * h;++ const edgeYLocalX = (0.5 - 0.5) * sx;+ const edgeYLocalY = (1 - 0.5) * sy;+ const edgeYAbsX = (px + edgeYLocalX * cos - edgeYLocalY * sin) * w;+ const edgeYAbsY = (py + edgeYLocalX * sin + edgeYLocalY * cos) * h;++ const distX = Math.hypot(edgeXAbsX - cx, edgeXAbsY - cy);+ const distY = Math.hypot(edgeYAbsX - cx, edgeYAbsY - cy);+ const outerRadius = Math.max(0.0001, distX, distY);++ const gradient = ctx.createRadialGradient(cx, cy, 0, cx, cy, outerRadius);++ addStopsTo(world, fill, gradient);+ return gradient;+}diff --git a/packages/runtime/src/systems/index.ts b/packages/runtime/src/systems/index.tsindex dffd197c..7df32c4a 100644--- a/packages/runtime/src/systems/index.ts+++ b/packages/runtime/src/systems/index.ts@@ -1,3 +1,4 @@// Deterministic systems. Browser-only systems (input, keyboard, camera, hud, dom) stay in apps/web.export * from './transform';export * from './motion';+export * from './gradients';diff --git a/packages/runtime/src/traits/media.ts b/packages/runtime/src/traits/media.tsindex 6ad6679d..048e4fa9 100644--- a/packages/runtime/src/traits/media.ts+++ b/packages/runtime/src/traits/media.ts@@ -4,6 +4,18 @@import { trait } from 'koota';+// Type-only imports: erased at compile time, so the traits ↔ media module+// cycle never materializes at runtime.+import type { HtmlHost } from '../media/html';+import type { SurfaceHost } from '../media/surface';+import type { ShaderHost } from '../media/shader';+import type { ImageDecoder } from '../media/image';+import type { VideoDecoderInstance } from '../media/video';+import type { SequenceDecoder } from '../media/sequence';+import type { AudioDecoder } from '../media/audio';+import type { CaptionDecoder } from '../media/caption/types';+import type { AudioBus } from '../media/audio-bus';+// Renderer-runtime state on mounted entities (never serialized).export type MountData = {props: Record<string, unknown>;@@ -12,26 +24,25 @@ export type MountData = {// Handle traits store live runtime instances (hosts, decoders, buses) and are// never serialized. They use AoS storage: entity.get(X) returns the instance-// directly. Handle types are opaque until engine/decoders and services move-// into src/media; tighten them to the concrete classes then.-type Handle = object;+// directly. Suffixed Handle so the trait and the class it stores (HtmlHost,+// AudioDecoder, ...) can share one export surface.-export const HtmlHost = trait(() => null as Handle | null);+export const HtmlHostHandle = trait(() => null as HtmlHost | null);-export const SurfaceHost = trait(() => null as Handle | null);+export const SurfaceHostHandle = trait(() => null as SurfaceHost | null);-export const ShaderHost = trait(() => null as Handle | null);+export const ShaderHostHandle = trait(() => null as ShaderHost | null);-export const ImageDecoder = trait(() => null as Handle | null);+export const ImageDecoderHandle = trait(() => null as ImageDecoder | null);-export const VideoDecoder = trait(() => null as Handle | null);+export const VideoDecoderHandle = trait(() => null as VideoDecoderInstance | null);-export const SequenceDecoder = trait(() => null as Handle | null);+export const SequenceDecoderHandle = trait(() => null as SequenceDecoder | null);-export const AudioDecoder = trait(() => null as Handle | null);+export const AudioDecoderHandle = trait(() => null as AudioDecoder | null);-export const CaptionDecoder = trait(() => null as Handle | null);+export const CaptionDecoderHandle = trait(() => null as CaptionDecoder | null);-export const AudioBus = trait(() => null as Handle | null);+export const AudioBusHandle = trait(() => null as AudioBus | null);export const Data = trait(() => null as MountData | null);diff --git a/packages/runtime/src/traits/text.ts b/packages/runtime/src/traits/text.tsindex d8400819..d8092397 100644--- a/packages/runtime/src/traits/text.ts+++ b/packages/runtime/src/traits/text.ts@@ -13,17 +13,20 @@ export const Chars = trait({ value: '' });// Text style properties, shared between a text node and its TextRange// sub-entities. On a text node this is the default style; on a range-// sub-entity it overrides.+// sub-entity it overrides. All fields default to undefined: a range override+// carries ONLY its explicitly set fields (a sparse store slot means "inherit+// from the node"), and the node-level fallbacks (16px, Inter, 400, ...) live+// in the text renderer's accessors.export const TextStyle = trait({- leading: 1.2,- fontSize: 16,- fontFamily: '',- fontWeight: '400',- fontStyle: FontStyle.NORMAL as FontStyle,- textAlign: TextAlign.LEFT as TextAlign,- textBaseline: TextBaseline.TOP as TextBaseline,- textCase: TextCase.ORIGINAL as TextCase,- letterSpacing: 0, // extra spacing between characters (px)+ leading: undefined as number | undefined,+ fontSize: undefined as number | undefined,+ fontFamily: undefined as string | undefined,+ fontWeight: undefined as string | undefined,+ fontStyle: undefined as FontStyle | undefined,+ textAlign: undefined as TextAlign | undefined,+ textBaseline: undefined as TextBaseline | undefined,+ textCase: undefined as TextCase | undefined,+ letterSpacing: undefined as number | undefined, // extra spacing between characters (px)});// Character range a TextStyle override applies to; end null = to the end.diff --git a/packages/runtime/src/utils/async.ts b/packages/runtime/src/utils/async.tsnew file mode 100644index 00000000..a369856e--- /dev/null+++ b/packages/runtime/src/utils/async.ts@@ -0,0 +1,35 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Attempt to execute a function and return the result or the error it threw.+ */+export function attempt<T>(fn: () => T): { result: T } | { error: Error } {+ try {+ return { result: fn() };+ } catch (error) {+ if (error instanceof Error) {+ return { error };+ }+ return { error: new Error('Unknown error') };+ }+}++export class AsyncMutex {+ currentPromise = Promise.resolve();++ async acquire() {+ let resolver: () => void;+ const nextPromise = new Promise<void>(resolve => {+ resolver = resolve;+ });++ const currentPromiseAlias = this.currentPromise;+ this.currentPromise = nextPromise;++ await currentPromiseAlias;++ return resolver!;+ }+}diff --git a/packages/runtime/src/utils/color.ts b/packages/runtime/src/utils/color.tsnew file mode 100644index 00000000..08b145c6--- /dev/null+++ b/packages/runtime/src/utils/color.ts@@ -0,0 +1,120 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Packed 0xRRGGBB color helpers (subset of the app's utils/color.ts that the+// renderer and shader uniforms need; UI-facing conversions stay app-side).++import { colord } from 'colord';++import { clamp } from '../math/common';++const NAMED_COLORS: Record<string, string> = {+ red: 'FF0000',+ green: '008000',+ blue: '0000FF',+ white: 'FFFFFF',+ black: '000000',+ yellow: 'FFFF00',+ cyan: '00FFFF',+ magenta: 'FF00FF',+ orange: 'FFA500',+ purple: '800080',+ pink: 'FFC0CB',+ gray: '808080',+ grey: '808080',+ brown: 'A52A2A',+ lime: '00FF00',+ navy: '000080',+ teal: '008080',+ maroon: '800000',+ olive: '808000',+ aqua: '00FFFF',+ coral: 'FF7F50',+ salmon: 'FA8072',+ gold: 'FFD700',+ silver: 'C0C0C0',+ indigo: '4B0082',+ violet: 'EE82EE',+ crimson: 'DC143C',+ tomato: 'FF6347',+ khaki: 'F0E68C',+ plum: 'DDA0DD',+ orchid: 'DA70D6',+ tan: 'D2B48C',+ beige: 'F5F5DC',+ ivory: 'FFFFF0',+ lavender: 'E6E6FA',+ turquoise: '40E0D0',+ chartreuse: '7FFF00',+ sienna: 'A0522D',+ peru: 'CD853F',+ firebrick: 'B22222',+ skyblue: '87CEEB',+ hotpink: 'FF69B4',+ mint: '3EB489',+};++/**+ * Parses any colord-compatible input (hex with or without `#`, short hex,+ * `rgb()`, `hsl()`, named colors, etc.) into a numeric hex color (0xRRGGBB).+ * Arbitrary strings fall back to Figma-style hex normalization: strip non-hex+ * characters, then repeat/pad to six digits (empty input returns null).+ */+export function parseColor(input: string): number | null {+ const trimmed = input.trim();+ if (trimmed.length === 0) return null;++ const named = NAMED_COLORS[trimmed.toLowerCase()];+ if (named) {+ return parseInt(named, 16);+ }++ const c = colord(trimmed);+ if (c.isValid()) {+ const { r, g, b } = c.toRgb();+ return rgbToColor(r, g, b);+ }++ const hex = trimmed.replace(/[^0-9A-Fa-f]/g, '');+ if (hex.length === 0) return null;++ let normalized: string;+ if (hex.length === 1) normalized = hex.repeat(6);+ else if (hex.length === 2) normalized = hex.repeat(3);+ else if (hex.length === 3) normalized = hex.split('').map((ch) => ch + ch).join('');+ else if (hex.length === 4) normalized = hex + '00';+ else if (hex.length === 5) normalized = hex + '0';+ else if (hex.length === 6) normalized = hex;+ else normalized = hex.slice(0, 6);++ return parseInt(normalized.toUpperCase(), 16);+}++/**+ * Converts a numeric hex color (0xRRGGBB) to a CSS hex string (#RRGGBB).+ */+export function colorToHex(color: number): string {+ return '#' + ((1 << 24) | (color & 0xFFFFFF)).toString(16).slice(1).toUpperCase();+}++/**+ * Packs RGB channel values (0-255) into a numeric hex color (0xRRGGBB).+ */+export function rgbToColor(r: number, g: number, b: number): number {+ const rc = clamp(Math.round(r), 0, 255);+ const gc = clamp(Math.round(g), 0, 255);+ const bc = clamp(Math.round(b), 0, 255);+ return (rc << 16) | (gc << 8) | bc;+}++/**+ * Unpacks a numeric hex color (0xRRGGBB) into RGB channel values (0-255).+ */+export function colorToRgb(color: number): { r: number; g: number; b: number } {+ return {+ r: (color >> 16) & 0xFF,+ g: (color >> 8) & 0xFF,+ b: color & 0xFF,+ };+}diff --git a/packages/runtime/src/utils/index.ts b/packages/runtime/src/utils/index.tsindex a458d2f2..6487bfec 100644--- a/packages/runtime/src/utils/index.ts+++ b/packages/runtime/src/utils/index.ts@@ -1,5 +1,7 @@// Pure helpers only: transition, captions follow. DOM-touching utils (cursor, interaction) stay in apps/web.export * from './assert';+export * from './async';+export * from './color';export * from './text';export * from './live-mounts';export * from './sort';diff --git a/packages/runtime/src/utils/text.ts b/packages/runtime/src/utils/text.tsindex b53cd454..46473328 100644--- a/packages/runtime/src/utils/text.ts+++ b/packages/runtime/src/utils/text.ts@@ -2,8 +2,25 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Text layout data model. The layout, measurement, and canvas render helpers-// join this file when engine/utils/text.ts moves into the runtime.+// Text layout, measurement, and canvas rendering (was engine/utils/text.ts).++import { store } from '../world/store';+import {+ COMPOSITE_OPERATIONS, PaintType, FontStyle, StrokeCap, StrokeJoin,+ TextAlign, TextBaseline, TextCase,+} from '../constants';+import {+ Size, Hidden, Paint, Color, Blur, Offset, Appearance, StrokeStyle,+ Chars, TextStyle, TextRange, TextCache, Cache, Computed, Camera,+ RenderSurface,+} from '../traits';+import { clamp } from '../math/common';+import { colorToHex } from './color';+import { createLinearGradient, createRadialGradient } from '../systems/gradients';++import type { Entity, World } from 'koota';++type Ctx = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;export type TokenOptions = {/**@@ -19,9 +36,9 @@ export type TokenOptions = {*/metrics: TextMetrics;/**- * Defines the style of the token+ * Defines the style of the token (TextRange sub-entities)*/- ranges: number[];+ ranges: Entity[];}export type Line = {@@ -31,10 +48,27 @@ export type Line = {height: number;};+type RenderSplit = {+ words: string[];+ ranges: Entity[];+};++/** Global offscreen canvas used solely for text measurement. */+let measureCanvas: OffscreenCanvas | null = null;+let measureCtx: OffscreenCanvasRenderingContext2D | null = null;++function getMeasureCtx(): OffscreenCanvasRenderingContext2D {+ if (!measureCtx) {+ measureCanvas = new OffscreenCanvas(1, 1);+ measureCtx = measureCanvas.getContext('2d', { willReadFrequently: true, alpha: true })!;+ }+ return measureCtx;+}+export class Token {public offset: number;public metrics: TextMetrics;- public ranges: number[];+ public ranges: Entity[];public chars: string;public width: number;public height: number;@@ -93,3 +127,559 @@ export class Token {this.line = { ...line };}}++function applyFont(ctx: Ctx, world: World, entity: Entity, ranges: Entity[]) {+ const size = getFontSize(world, entity, ranges);+ const family = getFontFamily(world, entity, ranges);+ const weight = getFontWeight(world, entity, ranges);+ const style = getFontStyle(world, entity, ranges);+ const baseline = getTextBaseline(world, entity, ranges);+ const spacing = getLetterSpacing(world, entity, ranges);++ const mappedStyle = FontStyle[style]!.toLowerCase();+ const mappedBaseline = TextBaseline[baseline]!.toLowerCase() as CanvasTextBaseline;++ ctx.font = `${mappedStyle} ${weight.toLowerCase()} ${size}px ${family}`.trim();+ ctx.textBaseline = mappedBaseline;+ ctx.letterSpacing = `${spacing}px`;+}++function applyStroke(ctx: Ctx, world: World, entity: Entity) {+ const stroke = store(world, StrokeStyle);+ const eid = entity.id();++ ctx.lineWidth = stroke.width[eid] ?? 1;+ ctx.lineJoin = StrokeJoin[stroke.join[eid] ?? 0]!.toLocaleLowerCase() as CanvasLineJoin;+ ctx.lineCap = StrokeCap[stroke.cap[eid] ?? 0]!.toLocaleLowerCase() as CanvasLineCap;+ ctx.miterLimit = stroke.miterLimit[eid] ?? 3;+}++function shapeTokens(world: World, entity: Entity): void {+ const computed = store(world, Computed);+ const textStyle = store(world, TextStyle);+ const eid = entity.id();+ const lines = store(world, TextCache).tokens[eid];+ const leading = textStyle.leading[eid] ?? 1;+ const textAlign = textStyle.textAlign[eid] ?? TextAlign.LEFT;+ const textBaseline = textStyle.textBaseline[eid] ?? TextBaseline.TOP;+ if (!lines) return;++ // Measure each line's width by summing word widths and spaces+ const lineWidths: number[] = lines.map(line => line.reduce((acc, word) => acc + word.width, 0));+ const lineHeights: number[] = lines.map(line => Math.max(...line.map(word => word.height)));++ // Find the maximum line width+ const maxLineWidth = Math.max(...lineWidths);++ // Calculate total height based on leading+ const totalHeight = lineHeights.reduce((acc, height, i) => acc + height * (i < lineHeights.length - 1 ? leading : 1), 0);++ // When Size is assigned, use Computed for positioning; otherwise set it from text dimensions+ if (!entity.has(Size) || !computed.width[eid] || !computed.height[eid]) {+ computed.width[eid] = Math.ceil(maxLineWidth);+ computed.height[eid] = Math.ceil(totalHeight);+ }++ const containerWidth = computed.width[eid]!;+ const containerHeight = computed.height[eid]!;++ // Calculate initial vertical offset based on text baseline and container height+ let offsetY = 0;+ if (textBaseline === TextBaseline.MIDDLE) {+ offsetY = (containerHeight - totalHeight) / 2;+ } else if (textBaseline === TextBaseline.BOTTOM) {+ offsetY = containerHeight - totalHeight;+ }++ // Calculate coordinates for each word+ const line = {+ offsetX: 0,+ offsetY,+ baseline: 0,+ height: 0,+ };++ for (let l = 0; l < lines.length; l++) {+ line.height = lineHeights[l]!;++ if (textAlign === TextAlign.LEFT) {+ line.offsetX = 0;+ } else if (textAlign === TextAlign.CENTER) {+ line.offsetX = (containerWidth - lineWidths[l]!) / 2;+ } else if (textAlign === TextAlign.RIGHT) {+ line.offsetX = containerWidth - lineWidths[l]!;+ }++ if (textBaseline === TextBaseline.TOP) {+ line.baseline = 0;+ } else if (textBaseline === TextBaseline.MIDDLE) {+ line.baseline = lineHeights[l]! / 2;+ } else if (textBaseline === TextBaseline.BOTTOM) {+ line.baseline = lineHeights[l]!;+ } else if (textBaseline === TextBaseline.ALPHABETIC) {+ const maxAscent = Math.max(...lines[l]!.map(word => word.metrics.fontBoundingBoxAscent));+ line.baseline = maxAscent || lineHeights[l]! * 0.75; // Fallback if no words in line+ }++ for (const word of lines[l]!) {+ word.setLine(line);+ }++ line.offsetY += lineHeights[l]! * leading;+ }+}++function tokenizeText(world: World, entity: Entity) {+ const ctx = getMeasureCtx();++ // Split text into segments based on style overrides first+ const lines: Token[][] = [[]];++ const maxWidth = store(world, Size).width[entity.id()] ?? Number.POSITIVE_INFINITY;++ let offset = 0;+ for (const { words, ranges } of createRenderSplits(world, entity)) {+ applyFont(ctx, world, entity, ranges);++ for (const word of words) {+ const splits = word.split('\n');++ for (let i = 0; i < splits.length; i++) {+ const caseValue = getTextCase(world, entity, ranges);+ const chars = transformText(splits[i]!, caseValue);++ const metrics = ctx.measureText(chars);++ // check if word is too wide for the current line but not the first word+ if (offset + metrics.width > maxWidth && offset > 0) {+ lines.push([]);+ offset = 0;+ }++ // add word to current line+ lines[lines.length - 1]!.push(new Token({ chars, ranges, metrics, offset }));+ offset += metrics.width;++ // add new line if not the last line+ if (i < splits.length - 1) {+ lines.push([]);+ offset = 0;+ }+ }+ }+ }++ if (!entity.has(TextCache)) entity.add(TextCache);+ store(world, TextCache).tokens[entity.id()] = lines;+}++/** Renders text tokens directly to the given canvas context. */+function renderTokens(ctx: Ctx, world: World, entity: Entity): void {+ const eid = entity.id();+ const lines = store(world, TextCache).tokens[eid];+ if (!lines) return;+ const words = lines.flat();++ const computed = store(world, Computed);+ const offsetStore = store(world, Offset);+ const blurStore = store(world, Blur);+ const colorStore = store(world, Color);+ const appearance = store(world, Appearance);+ const paintStore = store(world, Paint);++ const savedAlpha = ctx.globalAlpha;++ // Draw all text shadows+ {+ ctx.save();+ ctx.textAlign = 'start';+ ctx.textBaseline = 'top';++ // ctx.shadowBlur/OffsetX/OffsetY are in device-pixel space and are not+ // affected by the current transform, so scale them up to match the+ // content transform (camera * resolution).+ const camera = world.get(Camera);+ const resolution = world.get(RenderSurface)?.resolution ?? 1;+ const shadowScale = (camera?.a ?? 1) * resolution;++ for (const word of words) {+ applyFont(ctx, world, entity, word.ranges);++ const strokes = getStrokes(world, entity, word.ranges);+ const shadows = getShadows(world, entity, word.ranges);++ if (strokes.length) {+ applyStroke(ctx, world, entity);+ }++ // Draw shadows first (if any)+ for (const shadow of shadows) {+ if (shadow.has(Hidden)) continue;+ const sid = shadow.id();++ ctx.shadowOffsetX = (offsetStore.x[sid] ?? 0) * shadowScale;+ ctx.shadowOffsetY = (offsetStore.y[sid] ?? 0) * shadowScale;+ ctx.shadowBlur = (blurStore.value[sid] ?? 0) * shadowScale;+ ctx.shadowColor = colorToHex(colorStore.value[sid] ?? 0x000000);+ ctx.fillStyle = colorToHex(colorStore.value[sid] ?? 0x000000);+ ctx.globalAlpha = savedAlpha * (appearance.opacity[sid] ?? 1);++ if (strokes.length) {+ ctx.strokeText(word.chars, word.x, word.y);+ } else {+ ctx.fillText(word.chars, word.x, word.y);+ }+ }++ // Reset shadow properties if any shadows are applied+ if (shadows.length) {+ ctx.globalAlpha = savedAlpha;+ ctx.shadowColor = 'transparent';+ }+ }+ ctx.restore();+ }++ // Draw all text strokes+ {+ ctx.save();+ ctx.textAlign = 'start';+ ctx.textBaseline = 'top';++ for (const word of words) {+ const strokes = getStrokes(world, entity, word.ranges);+ if (!strokes.length) continue;++ applyFont(ctx, world, entity, word.ranges);+ applyStroke(ctx, world, entity);++ const w = computed.width[eid]!;+ const h = computed.height[eid]!;++ // Draw strokes (if any)+ for (const stroke of strokes) {+ if (stroke.has(Hidden)) continue;+ const sid = stroke.id();++ const savedCO = ctx.globalCompositeOperation;+ const blendMode = appearance.blendMode[sid] ?? 0;+ if (blendMode !== 0) {+ ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[blendMode]!;+ }+ ctx.globalAlpha = savedAlpha * (appearance.opacity[sid] ?? 1);++ const paintType = paintStore.value[sid];+ if (paintType === PaintType.LINEAR_GRADIENT) {+ ctx.strokeStyle = createLinearGradient(world, stroke, ctx, w, h);+ } else if (paintType === PaintType.RADIAL_GRADIENT) {+ ctx.strokeStyle = createRadialGradient(world, stroke, ctx, w, h);+ } else {+ ctx.strokeStyle = colorToHex(colorStore.value[sid] ?? 0x000000);+ }+ ctx.strokeText(word.chars, word.x, word.y);+ ctx.globalCompositeOperation = savedCO;+ }+ }+ ctx.restore();+ }++ // Draw all text fills+ {+ ctx.save();+ ctx.textAlign = 'start';+ ctx.textBaseline = 'top';++ for (const word of words) {+ const fills = getFills(world, entity, word.ranges);+ if (!fills.length) continue;++ applyFont(ctx, world, entity, word.ranges);++ const w = computed.width[eid]!;+ const h = computed.height[eid]!;++ for (const fill of fills) {+ if (fill.has(Hidden)) continue;+ const fid = fill.id();++ const savedCO = ctx.globalCompositeOperation;+ const blendMode = appearance.blendMode[fid] ?? 0;+ if (blendMode !== 0) {+ ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[blendMode]!;+ }+ ctx.globalAlpha = savedAlpha * (appearance.opacity[fid] ?? 1);++ const paintType = paintStore.value[fid];+ if (paintType === PaintType.LINEAR_GRADIENT) {+ ctx.fillStyle = createLinearGradient(world, fill, ctx, w, h);+ } else if (paintType === PaintType.RADIAL_GRADIENT) {+ ctx.fillStyle = createRadialGradient(world, fill, ctx, w, h);+ } else {+ ctx.fillStyle = colorToHex(colorStore.value[fid] ?? 0x000000);+ }+ ctx.fillText(word.chars, word.x, word.y);+ ctx.globalCompositeOperation = savedCO;+ }+ }+ ctx.restore();+ }+}++/** Render text tokens directly to the world's render surface. */+export function renderText(world: World, entity: Entity) {+ const ctx = world.get(RenderSurface)?.ctx;+ if (!ctx) return;+ tokenizeText(world, entity);+ shapeTokens(world, entity);+ renderTokens(ctx, world, entity);+}++/**+ * Splits text into segments based on overlapping text range entities.+ * Handles overlapping ranges by merging them.+ * @returns Array of RenderSplit objects with words and contributing ranges+ */+function createRenderSplits(world: World, entity: Entity): RenderSplit[] {+ const eid = entity.id();+ const chars = store(world, Computed).chars[eid] ?? store(world, Chars).value[eid] ?? '';++ const textRanges = store(world, Cache).textRanges[eid] ?? [];++ // Fast path: no styles+ if (textRanges.length === 0) {+ return [{+ words: tokenize(chars),+ ranges: [],+ }];+ }++ // Fast path: empty text+ if (chars.length === 0) {+ return [{+ words: [],+ ranges: [],+ }];+ }++ const rangeStore = store(world, TextRange);++ // Collect all unique boundary points+ const boundaries = new Set<number>();+ boundaries.add(0);+ boundaries.add(chars.length);++ for (const range of textRanges) {+ let start = rangeStore.start[range.id()] ?? 0;+ let end = rangeStore.end[range.id()] ?? chars.length;++ // Clamp boundaries to valid text range+ start = clamp(start, 0, chars.length);+ end = clamp(end, 0, chars.length);++ boundaries.add(start);+ boundaries.add(end);+ }++ // Sort boundaries in ascending order+ const sortedBoundaries = Array.from(boundaries).sort((a, b) => a - b);++ // Create segments between each pair of boundaries+ const segments: RenderSplit[] = [];++ for (let i = 0; i < sortedBoundaries.length - 1; i++) {+ const segStart = sortedBoundaries[i]!;+ const segEnd = sortedBoundaries[i + 1]!;++ // Skip empty segments+ if (segStart >= segEnd) continue;++ // Find all active styles for this segment+ const ranges: Entity[] = [];++ for (const range of textRanges) {+ let start = rangeStore.start[range.id()] ?? 0;+ let end = rangeStore.end[range.id()] ?? chars.length;++ // Clamp boundaries to valid text range+ start = clamp(start, 0, chars.length);+ end = clamp(end, 0, chars.length);++ // A style is active if segment falls within its range [start, end)+ // Using < for end to treat ranges as [start, end)+ if (start <= segStart && end > segStart) {+ ranges.push(range);+ }+ }++ segments.push({+ words: tokenize(chars.slice(segStart, segEnd)),+ ranges: ranges,+ });+ }++ return segments;+}++function tokenize(input: string): string[] {+ // Fast path for inputs without spaces+ if (input.indexOf(' ') === -1) {+ return [input];+ }++ // Use regex to match:+ // 1. Any characters up to and including a space+ // 2. OR any remaining characters to the end+ return input.match(/[^]*? |[^]+$/g) || [input];+}++export function transformText(text: string, textCase?: number): string {+ if (textCase == TextCase.LOWER) {+ return text.toLocaleLowerCase();+ }++ if (textCase == TextCase.UPPER) {+ return text.toUpperCase();+ }++ return text;+}++function getFills(world: World, entity: Entity, ranges: Entity[]): Entity[] {+ const cache = store(world, Cache);+ let value = cache.fills[entity.id()] ?? [];++ for (const range of ranges) {+ const rangeValue = cache.fills[range.id()];+ if (rangeValue?.length) {+ value = rangeValue;+ }+ }++ return value;+}++function getStrokes(world: World, entity: Entity, ranges: Entity[]): Entity[] {+ const cache = store(world, Cache);+ let value = cache.strokes[entity.id()] ?? [];++ for (const range of ranges) {+ const rangeValue = cache.strokes[range.id()];+ if (rangeValue?.length) {+ value = rangeValue;+ }+ }++ return value;+}++function getShadows(world: World, entity: Entity, ranges: Entity[]): Entity[] {+ const cache = store(world, Cache);+ let value = cache.shadows[entity.id()] ?? [];++ for (const range of ranges) {+ const rangeValue = cache.shadows[range.id()];+ if (rangeValue?.length) {+ value = rangeValue;+ }+ }++ return value;+}++function getTextCase(world: World, entity: Entity, ranges: Entity[]) {+ const textStyle = store(world, TextStyle);+ let value = textStyle.textCase[entity.id()] ?? 0;++ for (const range of ranges) {+ const rangeValue = textStyle.textCase[range.id()];+ if (rangeValue !== undefined) {+ value = rangeValue;+ }+ }++ return value;+}++function getFontSize(world: World, entity: Entity, ranges: Entity[]) {+ const textStyle = store(world, TextStyle);+ let value = textStyle.fontSize[entity.id()] ?? 16;++ for (const range of ranges) {+ const rangeValue = textStyle.fontSize[range.id()];+ if (rangeValue !== undefined) {+ value = rangeValue;+ }+ }++ return value;+}++function getFontFamily(world: World, entity: Entity, ranges: Entity[]) {+ const textStyle = store(world, TextStyle);+ let value = textStyle.fontFamily[entity.id()] || 'Inter';++ for (const range of ranges) {+ const rangeValue = textStyle.fontFamily[range.id()];+ if (rangeValue !== undefined) {+ value = rangeValue;+ }+ }++ return value;+}++function getFontWeight(world: World, entity: Entity, ranges: Entity[]) {+ const textStyle = store(world, TextStyle);+ let value = textStyle.fontWeight[entity.id()] ?? '400';++ for (const range of ranges) {+ const rangeValue = textStyle.fontWeight[range.id()];+ if (rangeValue !== undefined) {+ value = rangeValue;+ }+ }++ return value;+}++function getFontStyle(world: World, entity: Entity, ranges: Entity[]) {+ const textStyle = store(world, TextStyle);+ let value = textStyle.fontStyle[entity.id()] ?? FontStyle.NORMAL;++ for (const range of ranges) {+ const rangeValue = textStyle.fontStyle[range.id()];+ if (rangeValue !== undefined) {+ value = rangeValue;+ }+ }++ return value;+}++function getTextBaseline(world: World, entity: Entity, ranges: Entity[]) {+ const textStyle = store(world, TextStyle);+ let value = textStyle.textBaseline[entity.id()] ?? TextBaseline.TOP;++ for (const range of ranges) {+ const rangeValue = textStyle.textBaseline[range.id()];+ if (rangeValue !== undefined) {+ value = rangeValue;+ }+ }++ return value;+}++function getLetterSpacing(world: World, entity: Entity, ranges: Entity[]) {+ const textStyle = store(world, TextStyle);+ let value = textStyle.letterSpacing[entity.id()] ?? 0;++ for (const range of ranges) {+ const rangeValue = textStyle.letterSpacing[range.id()];+ if (rangeValue !== undefined) {+ value = rangeValue;+ }+ }++ return value;+}diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.tsindex 361cfaa2..902df892 100644--- a/packages/runtime/src/world/observers.ts+++ b/packages/runtime/src/world/observers.ts@@ -18,7 +18,7 @@import { store } from './store';import {- ChildOf, Deleted, Sequential, Group, Scene, Audio, Paint, AssetId,+ ChildOf, Deleted, Culled, Sequential, Group, Scene, Audio, Paint, AssetId,Delay, PlaybackRate, Trim, Keyframe, ItemIndex,Position, Offset, Rotation, Scale, UniformScale, Skew, Anchor, Flip,Appearance, Color, Blur, Volume, Effect, CornerRadius, MixedCornerRadius,@@ -26,6 +26,10 @@ import {} from '../traits';import { getParentEntity } from '../queries/hierarchy';import { rebuildCaches } from '../actions/cache';+import {+ disposeDecoders, disposeHtmlHosts, disposeSurfaceHosts, disposeShaderHosts,+ disconnectAudioBus,+} from '../media/dispose';import {reactToChildAttached, reactToChildDetached, reactToAssetChange,reactToPaintChange, recomputeEntityTimeRange, propagateTimeRangeDown,@@ -49,6 +53,7 @@ export function observeWorld(world: World): () => void {rebuildCaches(world, child, parent);propagateSize(world, child);resolveConstraintOffsets(world, child);+ disconnectAudioBus(world, child);reactToChildAttached(world, child);}));@@ -56,12 +61,26 @@ export function observeWorld(world: World): () => void {// still present in the old parent's queries, so it is excluded by hand.subs.push(world.onRemove(ChildOf('*'), (child, parent) => {rebuildCaches(world, child, parent, child);+ disconnectAudioBus(world, child);}));subs.push(world.onAdd(Deleted, (entity) => {// The Not(Deleted) cache queries already skip the tombstone.rebuildCaches(world, entity, getParentEntity(entity));reactToChildDetached(world, entity);+ // A tombstoned subtree keeps its records for undo, but its live media+ // handles must not linger.+ disposeDecoders(world, entity);+ disposeHtmlHosts(world, entity);+ disposeSurfaceHosts(world, entity);+ disposeShaderHosts(world, entity);+ disconnectAudioBus(world, entity);+ }));++ // Off-screen entities release their decoders (frame caches are the bulk+ // of media memory); the next resolve recreates them.+ subs.push(world.onAdd(Culled, (entity) => {+ disposeDecoders(world, entity);}));subs.push(world.onRemove(Deleted, (entity) => {diff --git a/packages/runtime/tsconfig.json b/packages/runtime/tsconfig.jsonindex d3ad84ef..516eda7b 100644--- a/packages/runtime/tsconfig.json+++ b/packages/runtime/tsconfig.json@@ -3,7 +3,8 @@"compilerOptions": {"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo","erasableSyntaxOnly": false,- "lib": ["es2024", "dom"]+ "lib": ["es2024", "DOM", "DOM.Iterable"],+ "types": ["@webgpu/types"]},"include": ["src"]}