Skip to content

The VS Code of video editing, built for agents.

TypeScript

Update runtime package to include encoder system and enhance rendering capabilities

konstantin-paulus664eb897c8e4
diff --git a/package-lock.json b/package-lock.json
index 90d8070c..6a1de354 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -593,6 +593,10 @@
"resolved": "apps/desktop",
"link": true
},
+ "node_modules/@diffusionstudio/encoder": {
+ "resolved": "packages/encoder",
+ "link": true
+ },
"node_modules/@diffusionstudio/jsx": {
"resolved": "packages/jsx",
"link": true
@@ -12300,6 +12304,21 @@
"url": "https://github.com/sponsors/colinhacks"
}
},
+ "packages/encoder": {
+ "name": "@diffusionstudio/encoder",
+ "version": "0.1.0",
+ "license": "MPL-2.0",
+ "dependencies": {
+ "@diffusionstudio/runtime": "*",
+ "koota": "^0.6.6",
+ "mediabunny": "^1.32.2"
+ },
+ "devDependencies": {
+ "@types/audioworklet": "^0.0.92",
+ "@webgpu/types": "^0.1.71",
+ "typescript": "~5.9.3"
+ }
+ },
"packages/jsx": {
"name": "@diffusionstudio/jsx",
"version": "0.1.0",
diff --git a/packages/encoder/package.json b/packages/encoder/package.json
new file mode 100644
index 00000000..319639ac
--- /dev/null
+++ b/packages/encoder/package.json
@@ -0,0 +1,29 @@
+{
+ "name": "@diffusionstudio/encoder",
+ "version": "0.1.0",
+ "description": "Offline video/audio/image encoding over @diffusionstudio/runtime worlds via mediabunny. Needs a browser-grade context (OffscreenCanvas, OfflineAudioContext, AudioWorklet, SharedArrayBuffer).",
+ "license": "MPL-2.0",
+ "type": "module",
+ "sideEffects": false,
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "scripts": {
+ "check": "tsc --noEmit"
+ },
+ "keywords": [
+ "encoder",
+ "video",
+ "webcodecs"
+ ],
+ "dependencies": {
+ "@diffusionstudio/runtime": "*",
+ "koota": "^0.6.6",
+ "mediabunny": "^1.32.2"
+ },
+ "devDependencies": {
+ "@types/audioworklet": "^0.0.92",
+ "@webgpu/types": "^0.1.71",
+ "typescript": "~5.9.3"
+ }
+}
diff --git a/packages/encoder/src/buffer.ts b/packages/encoder/src/buffer.ts
new file mode 100644
index 00000000..72705ae7
--- /dev/null
+++ b/packages/encoder/src/buffer.ts
@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { BufferTarget, StreamTarget, Target } from 'mediabunny';
+
+import type { ContainerFormat, WritableFileTarget } from './types';
+
+export class TargetBuffer {
+ public target: Target;
+ public fastStart: false | 'in-memory';
+ public handle: FileSystemFileHandle | WritableFileTarget | undefined;
+
+ public constructor(
+ target: StreamTarget | BufferTarget,
+ fastStart: false | 'in-memory',
+ handle: FileSystemFileHandle | WritableFileTarget | undefined
+ ) {
+ this.target = target;
+ this.fastStart = fastStart;
+ this.handle = handle;
+ }
+
+ public static async create(
+ handle?: FileSystemFileHandle | WritableFileTarget
+ ): Promise<TargetBuffer> {
+ if (handle) {
+ const fileStream = await handle.createWritable();
+ return new TargetBuffer(new StreamTarget(fileStream, { chunked: true }), false, handle);
+ }
+
+ return new TargetBuffer(new BufferTarget(), 'in-memory', undefined);
+ }
+
+ public async close(format: ContainerFormat = 'mp4'): Promise<Blob | undefined> {
+ if (this.fastStart === 'in-memory' && this.target instanceof BufferTarget) {
+ if (!this.target.buffer) return;
+ return new Blob([this.target.buffer], { type: mimeTypes[format] });
+ }
+
+ return undefined;
+ }
+}
+
+const mimeTypes: Record<ContainerFormat, string> = {
+ mp4: 'video/mp4',
+ webm: 'video/webm',
+ ogg: 'audio/ogg',
+ mov: 'video/quicktime',
+};
diff --git a/packages/encoder/src/encoder.ts b/packages/encoder/src/encoder.ts
new file mode 100644
index 00000000..7ea3404d
--- /dev/null
+++ b/packages/encoder/src/encoder.ts
@@ -0,0 +1,511 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import {
+ CanvasSource,
+ Output,
+ AudioSample,
+ AudioSampleSource,
+} from 'mediabunny';
+import { Not, Or } from 'koota';
+import {
+ createRuntimeWorld, switchActiveScene, serializeEntity, cloneFromRecords,
+ getEntityTree, getDocument, propagateTimeRangeDown, framesToSeconds,
+ assert, store, disposeDecoders,
+ playbackSystem, motionSystem, transformSystem, renderSystem,
+ AudioBus, AudioBusHandle,
+ ChildOf, Deleted, Geometry, Group, Paint, Workarea, Playback,
+ AudioPlayback, Computed, Delay, Trim, Transition, Keyframe, Animation,
+ Position, Offset, Rotation, Scale, Skew,
+ Project, Mode, Time, FrameRate, Camera, RenderSurface, AudioEngine,
+ Assets, Fonts, FramePromises,
+} from '@diffusionstudio/runtime';
+
+import { TargetBuffer } from './buffer';
+import { createOutputFormat } from './format';
+import { createRenderEventDetail } from './utils';
+
+import type { World } from 'koota';
+import type { EncoderConfig } from './interfaces';
+import type { ExportResult } from './types';
+
+export async function createEncoder(sourceWorld: World, config: EncoderConfig) {
+ const sourceComputed = store(sourceWorld, Computed);
+ const sourceScene = config.scene;
+ const sourceSceneId = sourceScene.id();
+ const sourceFrameRate = sourceWorld.get(FrameRate)?.value ?? 30;
+ const frameRate = config.video?.fps ?? sourceFrameRate;
+ const numberOfChannels = config.audio?.numberOfChannels ?? 2;
+ const sampleRate = config.audio?.sampleRate ?? 48000;
+ const sceneWidth = sourceComputed.width[sourceSceneId]!;
+ const sceneHeight = sourceComputed.height[sourceSceneId]!;
+ const sceneEnd = sourceComputed.end[sourceSceneId]!;
+
+ // Honor the scene's Workarea trait: render only the frames between
+ // Workarea.start and Workarea.end instead of the full scene duration.
+ const hasWorkarea = sourceScene.has(Workarea);
+ const sourceWorkarea = store(sourceWorld, Workarea);
+ const workareaStart = hasWorkarea
+ ? Math.max(0, Math.min(sceneEnd, sourceWorkarea.start[sourceSceneId] ?? 0))
+ : 0;
+ const workareaEnd = hasWorkarea
+ ? Math.max(workareaStart, Math.min(sceneEnd, sourceWorkarea.end[sourceSceneId] ?? sceneEnd))
+ : sceneEnd;
+ const workareaFrames = workareaEnd - workareaStart;
+
+ const playheadStartSeconds = framesToSeconds(workareaStart, sourceFrameRate);
+ const duration = framesToSeconds(workareaFrames, sourceFrameRate);
+ const audioBitrate = config.audio?.bitrate ?? 128e3;
+ const videoBitrate = config.video?.bitrate ?? 10e6;
+ const audioCodec = config.audio?.codec ?? 'aac';
+ const resolution = config.video?.resolution ?? 1080;
+ const scale = Math.round(resolution * 1e6 / sceneHeight) / 1e6;
+ const width = Math.round(sceneWidth * scale / 2) * 2;
+ const height = Math.round(sceneHeight * scale / 2) * 2;
+ const videoCodec = config.video?.codec ?? 'avc';
+ const containerFormat = config.format ?? 'mp4';
+ const audioEnabled = config.audio?.enabled ?? true;
+ const videoEnabled = config.video?.enabled ?? true;
+ const frameDuration = 1 / frameRate;
+
+ if (config.format === 'ogg') {
+ config.audio = { ...config.audio, enabled: true };
+ config.video = { ...config.video, enabled: false };
+ }
+
+ const sceneSubtree = getEntityTree(sourceWorld, sourceScene);
+ const subtreeRecords = sceneSubtree.map(entity => serializeEntity(entity));
+
+ // Offscreen canvas
+ const offscreenCanvas = new OffscreenCanvas(width, height);
+ const offscreenCtx = offscreenCanvas.getContext('2d')!;
+
+ // Offline audio context
+ const offlineAudioCtx = new OfflineAudioContext(
+ numberOfChannels,
+ Math.ceil(duration * sampleRate),
+ sampleRate,
+ );
+
+ // Build a fresh offline world that shares assets with the source. Fonts are
+ // already loaded into the document; copying the array just tracks them as
+ // loaded so the new world doesn't try to re-add them.
+ const world = createRuntimeWorld(sourceWorld.get(Project)?.id ?? '');
+ world.set(Mode, { value: videoEnabled ? 'offline-video' : 'offline-audio' });
+ world.set(Assets, sourceWorld.get(Assets)!);
+ world.set(Fonts, { list: [...(sourceWorld.get(Fonts)?.list ?? [])] });
+ world.set(RenderSurface, { canvas: offscreenCanvas, ctx: offscreenCtx, resolution: scale });
+ world.set(AudioEngine, { context: offlineAudioCtx });
+ world.set(Camera, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 });
+ world.set(FrameRate, { value: frameRate });
+ world.set(FramePromises, { list: [] });
+
+ const eidMap = cloneFromRecords(world, subtreeRecords);
+ const scene = eidMap.get(sourceScene);
+ assert(scene !== undefined, 'Failed to clone scene subtree');
+
+ rescaleFrameTimestamps(world, sourceFrameRate, frameRate);
+ recomputeAllTimeRanges(world);
+ normalizeSceneTransform(world, scene.id());
+
+ switchActiveScene(world, scene);
+
+ const computed = store(world, Computed);
+ const playback = store(world, Playback);
+ // The workarea (Workarea trait) is runtime-only and never serialized, so it
+ // isn't cloned into the offline world — nothing to strip here. The encoder
+ // drives the playhead explicitly within the [workareaStart, workareaEnd]
+ // window computed from the source scene above.
+ const sceneId = scene.id();
+ computed.localTimeInSeconds[sceneId] = playheadStartSeconds;
+ computed.localTime[sceneId] = Math.round(playheadStartSeconds * frameRate);
+ playback.playing[sceneId] = true;
+ playback.loop[sceneId] = false;
+ playback.speed[sceneId] = 1;
+ // Anchor audio scheduling: context starts at t=0 but the playhead starts at
+ // playheadStartSeconds, so audioDelay = -playheadStartSeconds shifts each
+ // entity's scheduled audio back into the encoded window.
+ if (!scene.has(AudioPlayback)) scene.add(AudioPlayback);
+ const audioPlayback = store(world, AudioPlayback);
+ audioPlayback.contextOffsetInSeconds[sceneId] = 0;
+ audioPlayback.timelineOffsetInSeconds[sceneId] = playheadStartSeconds;
+
+ // Re-execute any mounts into this offline world so it owns its own reactive
+ // graphs + runtime hosts (surface/html). The per-frame playbackSystem then
+ // drives them via the Mounts trait, so ticker-animated surfaces render.
+ const mounts = videoEnabled ? (await config.realizeMounts?.(world)) ?? null : null;
+
+ // Set up mediabunny output
+ const buffer = await TargetBuffer.create(config.target);
+ const format = await createOutputFormat(buffer, config.format);
+ const output = new Output({ format, target: buffer.target });
+
+ if (config.comment) {
+ output.setMetadataTags({ comment: config.comment });
+ }
+
+ const audioSource = new AudioSampleSource({
+ codec: audioCodec,
+ bitrate: audioBitrate,
+ });
+
+ const sceneFills = [...world.query(Paint, ChildOf(scene), Not(Deleted))];
+
+ const videoSource = new CanvasSource(offscreenCanvas, {
+ codec: videoCodec,
+ bitrate: videoBitrate,
+ latencyMode: 'quality',
+ alpha: containerFormat === 'webm' && !sceneFills.length
+ ? 'keep'
+ : 'discard',
+ });
+
+ if (audioEnabled) {
+ output.addAudioTrack(audioSource);
+ }
+
+ if (videoEnabled) {
+ output.addVideoTrack(videoSource);
+ }
+
+ // Audio worklet streams render-quantum chunks back to the main thread so
+ // we can hand them to mediabunny in step with the visual frame loop.
+ const audioWorkletUrl = createAudioWorkletUrl();
+ await offlineAudioCtx.audioWorklet.addModule(audioWorkletUrl);
+
+ const sharedBuffer = new SharedArrayBuffer(4);
+ const sharedUint32Array = new Uint32Array(sharedBuffer);
+ const mixNode = offlineAudioCtx.createGain();
+
+ const sinkNode = new AudioWorkletNode(offlineAudioCtx, 'sink', {
+ channelCount: numberOfChannels,
+ channelCountMode: 'explicit',
+ numberOfInputs: 1,
+ numberOfOutputs: 1,
+ outputChannelCount: [numberOfChannels],
+ processorOptions: {
+ buffer: sharedUint32Array,
+ },
+ });
+
+ mixNode.connect(sinkNode);
+ sinkNode.connect(offlineAudioCtx.destination);
+
+ // connect the scene bus to the mix node
+ const sceneBus = new AudioBus(world, scene);
+ scene.add(AudioBusHandle);
+ scene.set(AudioBusHandle, sceneBus);
+ sceneBus.connect(mixNode);
+
+ let sampleIndex = 0;
+ let audioSourcePromise: Promise<void> | null = null;
+
+ const allAudioReceived = Promise.withResolvers<void>();
+
+ sinkNode.port.onmessage = event => {
+ if (output?.state === 'canceled') {
+ return;
+ }
+
+ const planarBuffer = event.data as Float32Array;
+
+ if (planarBuffer.length === 0) {
+ allAudioReceived.resolve();
+ return;
+ }
+
+ const sampleCount = planarBuffer.length / numberOfChannels;
+
+ const audioSample = new AudioSample({
+ data: planarBuffer,
+ format: 'f32-planar',
+ numberOfChannels,
+ sampleRate,
+ timestamp: sampleIndex / sampleRate,
+ });
+ audioSourcePromise = audioSource.add(audioSample).finally(() => {
+ audioSample.close();
+ });
+
+ sampleIndex += sampleCount;
+ };
+
+ const emitProgress = createThrottledCallback(config.onProgress, 1000 / 3); // 3 times per second
+
+ let canceled = false;
+ const cancel = () => (canceled = true);
+
+ const render = async (): Promise<ExportResult> => {
+ try {
+ await output.start();
+ const start = performance.now();
+ const startTime = start;
+ const totalFrames = Math.floor(duration * frameRate);
+
+ let audioRenderingDone = false;
+ let audioRenderingCompleted: Promise<AudioBuffer> | null = null;
+ if (audioEnabled) {
+ audioRenderingCompleted = offlineAudioCtx.startRendering();
+ audioRenderingCompleted.then(() => { audioRenderingDone = true; });
+ }
+
+ let lastAudioSampleCount = 0;
+
+ for (let frame = 0; frame < totalFrames; frame++) {
+ if (canceled) {
+ await output.cancel();
+ Atomics.store(sharedUint32Array, 0, 2 ** 31 - 1);
+ return { type: 'canceled' };
+ }
+
+ const timestamp = frame * frameDuration;
+ const playheadSeconds = playheadStartSeconds + timestamp;
+
+ // advancePlayhead is a no-op in offline mode; the encoder owns the
+ // playhead and writes both seconds + frames before each tick.
+ computed.localTimeInSeconds[sceneId] = playheadSeconds;
+ computed.localTime[sceneId] = Math.round(playheadSeconds * frameRate);
+ const time = world.get(Time)!;
+ world.set(Time, { delta: frameDuration * 1000, now: time.now + frameDuration * 1000 });
+
+ {
+ playbackSystem(world);
+ await resolverSystem(world);
+ motionSystem(world);
+ }
+
+ if (videoEnabled) {
+ // only visual systems
+ transformSystem(world);
+ renderSystem(world);
+ }
+
+ if (videoEnabled) {
+ await videoSource.add(timestamp, frameDuration);
+ }
+
+ if (audioEnabled) {
+ await audioSourcePromise;
+
+ while (Atomics.load(sharedUint32Array, 0) > sampleRate && !audioRenderingDone) {
+ await new Promise(resolve => setTimeout(resolve, 0));
+ }
+
+ const totalAudioSampleCount = Math.floor(timestamp * sampleRate);
+ const diff = totalAudioSampleCount - lastAudioSampleCount;
+ Atomics.add(sharedUint32Array, 0, diff);
+ lastAudioSampleCount = totalAudioSampleCount;
+ }
+
+ emitProgress(createRenderEventDetail(frame, totalFrames, startTime));
+ }
+
+ console.info(
+ `Encoded frames at ${((totalFrames * 1000) / (performance.now() - start)).toFixed(2)}FPS`
+ );
+
+ if (audioEnabled) {
+ assert(audioRenderingCompleted !== null, 'Audio rendering was not started');
+
+ Atomics.add(
+ sharedUint32Array,
+ 0,
+ Math.ceil(duration * sampleRate) + sampleRate - lastAudioSampleCount,
+ );
+
+ await audioRenderingCompleted;
+
+ sinkNode.port.postMessage(null);
+ await allAudioReceived.promise;
+ }
+
+ console.info('Finalizing file');
+
+ await output.finalize();
+
+ console.info('Export complete');
+
+ return {
+ type: 'success',
+ data: await buffer.close(containerFormat),
+ };
+ } catch (e) {
+ return {
+ type: 'error',
+ error: e instanceof Error ? e : new Error('Unknown error'),
+ };
+ } finally {
+ URL.revokeObjectURL(audioWorkletUrl);
+ // Free the graphs + hosts realized above (this offline world is
+ // discarded, so nothing else disposes them — an HtmlHost otherwise
+ // leaks a canvas on document.body).
+ mounts?.disposeAll();
+ // Release decoder file handles/caches, then the world slot itself
+ // (koota caps live worlds at 16, so throwaway offline worlds must
+ // be destroyed — bitecs could just drop them).
+ disposeDecoders(world, getDocument(world));
+ world.destroy();
+ }
+ };
+
+ return {
+ render,
+ cancel,
+ };
+}
+
+
+function audioWorkletCode() {
+ // eslint-disable-next-line no-undef
+ class SinkProcessor extends AudioWorkletProcessor {
+ buffer: Uint32Array;
+
+ constructor(options: {
+ processorOptions: {
+ buffer: Uint32Array;
+ };
+ }) {
+ super();
+
+ this.buffer = options.processorOptions.buffer;
+
+ this.port.onmessage = () => {
+ this.port.postMessage(new Float32Array(0));
+ };
+ }
+
+ process(inputs: Float32Array[][], outputs: Float32Array[][]) {
+ const input = inputs[0];
+ const output = outputs[0];
+ const renderQuantum = output[0].length;
+
+ while (Atomics.load(this.buffer, 0) < renderQuantum);
+
+ const planarBuffer = new Float32Array(output.length * output[0].length);
+ for (let channel = 0; channel < Math.min(input.length, output.length); channel++) {
+ planarBuffer.set(input[channel], channel * output[0].length);
+ }
+
+ this.port.postMessage(planarBuffer, [planarBuffer.buffer]);
+
+ Atomics.sub(this.buffer, 0, renderQuantum);
+
+ return true;
+ }
+ }
+
+ // eslint-disable-next-line no-undef
+ registerProcessor('sink', SinkProcessor);
+}
+
+function createAudioWorkletUrl(): string {
+ const blob = new Blob([`(${audioWorkletCode.toString()})()`], { type: 'application/javascript' });
+ return URL.createObjectURL(blob);
+}
+
+function createThrottledCallback<T>(callback: ((value: T) => void) | undefined, intervalMs: number): (value: T) => void {
+ let lastCalledAt = -Infinity;
+
+ return (value: T) => {
+ const now = performance.now();
+ if (now - lastCalledAt < intervalMs) {
+ return;
+ }
+
+ lastCalledAt = now;
+ callback?.(value);
+ };
+}
+
+function rescaleFrameTimestamps(
+ world: World,
+ sourceFrameRate: number,
+ targetFrameRate: number,
+): void {
+ if (sourceFrameRate === targetFrameRate) return;
+
+ const ratio = targetFrameRate / sourceFrameRate;
+ const rescale = (v: number) => Math.round(v * ratio);
+
+ const delay = store(world, Delay);
+ const trim = store(world, Trim);
+ const transition = store(world, Transition);
+ const keyframe = store(world, Keyframe);
+ const animation = store(world, Animation);
+
+ for (const entity of world.query(Or(Delay, Trim, Transition, Keyframe, Animation))) {
+ const eid = entity.id();
+ if (entity.has(Delay)) {
+ delay.value[eid] = rescale(delay.value[eid] ?? 0);
+ }
+ if (entity.has(Trim)) {
+ trim.start[eid] = rescale(trim.start[eid] ?? 0);
+ trim.end[eid] = rescale(trim.end[eid] ?? 0);
+ }
+ if (entity.has(Transition)) {
+ transition.duration[eid] = rescale(transition.duration[eid] ?? 0);
+ }
+ if (entity.has(Keyframe)) {
+ keyframe.time[eid] = rescale(keyframe.time[eid] ?? 0);
+ }
+ if (entity.has(Animation)) {
+ animation.duration[eid] = rescale(animation.duration[eid] ?? 0);
+ animation.delay[eid] = rescale(animation.delay[eid] ?? 0);
+ }
+ }
+}
+
+function normalizeSceneTransform(world: World, sceneId: number): void {
+ const position = store(world, Position);
+ const offset = store(world, Offset);
+ const rotation = store(world, Rotation);
+ const scaleStore = store(world, Scale);
+ const skew = store(world, Skew);
+ position.x[sceneId] = 0;
+ position.y[sceneId] = 0;
+ offset.x[sceneId] = 0;
+ offset.y[sceneId] = 0;
+ rotation.value[sceneId] = 0;
+ scaleStore.x[sceneId] = 1;
+ scaleStore.y[sceneId] = 1;
+ skew.x[sceneId] = 0;
+ skew.y[sceneId] = 0;
+ // computeLocalMatrix reads the Computed.* mirror, not the raw traits. The
+ // base traits are populated via entity.set during the clone, which fires
+ // the observer that copies the (canvas) position into Computed. Writing
+ // the raw stores above bypasses that observer, so the Computed mirror must
+ // be reset here too — otherwise the scene renders at its canvas position
+ // instead of centered in the export.
+ const computed = store(world, Computed);
+ computed.positionX[sceneId] = 0;
+ computed.positionY[sceneId] = 0;
+ computed.offsetX[sceneId] = 0;
+ computed.offsetY[sceneId] = 0;
+ computed.rotation[sceneId] = 0;
+ computed.scaleX[sceneId] = 1;
+ computed.scaleY[sceneId] = 1;
+ computed.skewX[sceneId] = 0;
+ computed.skewY[sceneId] = 0;
+}
+
+export async function resolverSystem(world: World) {
+ const promises = world.get(FramePromises)?.list;
+ if (promises?.length) {
+ await Promise.all(promises.filter(promise => promise !== null));
+ world.set(FramePromises, { list: [] });
+ }
+}
+
+/**
+ * Refresh every entity's accumulated delay + time bounds after raw store
+ * rewrites. propagateTimeRangeDown recomputes top-down and then re-derives
+ * each parent after its children (bitecs used a two-pass Hierarchy(ChildOf)
+ * walk; koota has no hierarchy-ordered queries).
+ */
+export function recomputeAllTimeRanges(world: World): void {
+ const document = getDocument(world);
+ for (const node of world.query(Or(Geometry, Group), ChildOf(document), Not(Deleted))) {
+ propagateTimeRangeDown(world, node);
+ }
+}
diff --git a/packages/encoder/src/format.ts b/packages/encoder/src/format.ts
new file mode 100644
index 00000000..1b831363
--- /dev/null
+++ b/packages/encoder/src/format.ts
@@ -0,0 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { TargetBuffer } from './buffer';
+import type { ContainerFormat } from './types';
+
+/**
+ * Lazily imports the output format based on the format specified
+ */
+export async function createOutputFormat(buffer: TargetBuffer, format?: ContainerFormat) {
+ if (format == 'webm') {
+ const { WebMOutputFormat } = await import('mediabunny');
+ return new WebMOutputFormat();
+ } else if (format == 'ogg') {
+ const { OggOutputFormat } = await import('mediabunny');
+ return new OggOutputFormat();
+ } else if (format == 'mov') {
+ const { MovOutputFormat } = await import('mediabunny');
+ return new MovOutputFormat();
+ } else {
+ const { Mp4OutputFormat } = await import('mediabunny');
+ return new Mp4OutputFormat({ fastStart: buffer.fastStart });
+ }
+}
diff --git a/packages/encoder/src/image-encoder.ts b/packages/encoder/src/image-encoder.ts
new file mode 100644
index 00000000..0f89df65
--- /dev/null
+++ b/packages/encoder/src/image-encoder.ts
@@ -0,0 +1,241 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Not, Or } from 'koota';
+import {
+ createRuntimeWorld, createEntity, appendChild, serializeEntity,
+ cloneFromRecords, getEntityTree, getDocument, framesToSeconds,
+ formatTimecode, assert, store, disposeDecoders,
+ playbackSystem, motionSystem, transformSystem, renderSystem,
+ ChildOf, Deleted, Geometry, Group, Hidden, IsMask, Muted, Culled,
+ ClipsContent, Playback, Computed, WorldBounds,
+ Project, Mode, Camera, RenderSurface, AudioEngine, Assets, Fonts,
+ FramePromises, Time, FrameRate,
+} from '@diffusionstudio/runtime';
+
+import { resolverSystem, recomputeAllTimeRanges } from './encoder';
+
+import type { Entity, World } from 'koota';
+import type { AABB } from '@diffusionstudio/runtime';
+import type { ImageEncoderConfig } from './interfaces';
+
+/** One capture: the PNG plus the timecode of the frame rendered, e.g. `01s15f`. */
+export type CapturedImage = { base64: string; timecode: string };
+
+export type ImageExportResult =
+ | { type: 'success'; data: CapturedImage[] }
+ | { type: 'canceled' }
+ | { type: 'error'; error: Error };
+
+/**
+ * Renders single frames of one entity into standalone PNGs (base64, no
+ * data-url prefix). The entity's subtree is cloned into a fresh offline
+ * world and drawn at the top level: the canvas is sized to the union of the
+ * entity's world-space AABBs across the requested frames and the camera
+ * shifts that box onto the canvas, so the node is fully visible regardless
+ * of where it sat in its source scene.
+ */
+export async function createImageEncoder(sourceWorld: World, config: ImageEncoderConfig) {
+ assert(config.frames.length > 0, 'No frames requested');
+
+ const subtree = getEntityTree(sourceWorld, config.entity);
+ const records = subtree.map(entity => serializeEntity(entity));
+
+ // Re-root the node. Its parent isn't part of the clone, so a dangling
+ // ChildOf would attach to an arbitrary entity in the new world; Delay and
+ // Hidden only mean something on the timeline it was lifted out of.
+ const rootRecord = records.find(record => record.eid === config.entity);
+ assert(rootRecord !== undefined, `No such entity: ${config.entity}`);
+ delete rootRecord.ChildOf;
+ delete rootRecord.Delay;
+ delete rootRecord.Hidden;
+
+ const offscreenCanvas = new OffscreenCanvas(2, 2);
+ const offscreenCtx = offscreenCanvas.getContext('2d')!;
+
+ // Never started — image capture is video-only; the context just satisfies
+ // world construction and lazy audio-bus wiring.
+ const offlineAudioCtx = new OfflineAudioContext(2, 1, 48000);
+
+ const sourceFrameRate = sourceWorld.get(FrameRate)?.value ?? 30;
+ const world = createRuntimeWorld(sourceWorld.get(Project)?.id ?? '');
+ world.set(Mode, { value: 'offline-video' });
+ world.set(Assets, sourceWorld.get(Assets)!);
+ world.set(Fonts, { list: [...(sourceWorld.get(Fonts)?.list ?? [])] });
+ world.set(RenderSurface, { canvas: offscreenCanvas, ctx: offscreenCtx, resolution: 1 });
+ world.set(AudioEngine, { context: offlineAudioCtx });
+ world.set(Camera, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 });
+ world.set(FrameRate, { value: sourceFrameRate });
+ world.set(FramePromises, { list: [] });
+
+ const eidMap = cloneFromRecords(world, records);
+ const root = eidMap.get(config.entity);
+ assert(root !== undefined, 'Failed to clone entity subtree');
+
+ const mounts = (await config.realizeMounts?.(world)) ?? null;
+
+ const cloned = [...eidMap.values()];
+ const computed = store(world, Computed);
+ const playback = store(world, Playback);
+ const worldBounds = store(world, WorldBounds);
+
+ // Mute everything so the playback system never initializes audio decoders.
+ for (const entity of cloned) {
+ entity.add(Muted);
+ }
+
+ // The motion system only samples parented entities, so the node's own
+ // keyframes would be skipped as a top-level root. A synthetic group above
+ // it owns the playhead instead and keeps the node an ordinary child.
+ const stage = createEntity(world);
+ stage.add(Group);
+ appendChild(world, root, stage);
+
+ recomputeAllTimeRanges(world);
+
+ // Map "frame 0 = first visible frame": with Delay stripped the node's
+ // computed start is 0 unless it carries a Trim, whose start offsets it.
+ const startFrame = computed.start[root.id()] ?? 0;
+
+ // Entities in the clone that own a timeline clock
+ const clocks = [...world.query(Playback, Not(Deleted))];
+
+ const seek = (frame: number) => {
+ const stageFrame = startFrame + frame;
+ const stageSeconds = framesToSeconds(stageFrame, sourceFrameRate);
+ computed.localTime[stage.id()] = stageFrame;
+ computed.localTimeInSeconds[stage.id()] = stageSeconds;
+
+ for (const clock of clocks) {
+ computed.localTime[clock.id()] = stageFrame;
+ computed.localTimeInSeconds[clock.id()] = stageSeconds;
+ playback.playing[clock.id()] = true;
+ }
+
+ const delta = 1000 / sourceFrameRate;
+ const time = world.get(Time)!;
+ world.set(Time, { delta, now: time.now + delta });
+
+ // Cull flags derived from a stale canvas/camera would make the motion
+ // system skip entities and keep decoders idle — clear them every tick.
+ for (const entity of cloned) {
+ if (entity.has(Culled)) entity.remove(Culled);
+ }
+
+ playbackSystem(world);
+ };
+
+ // Union an entity's world-space AABB into `bounds`, descending so children
+ // that overflow a leaf's own rect still count — except where content is
+ // clipped. Group bounds already aggregate their children.
+ const measure = (entity: Entity, bounds: AABB): void => {
+ if (entity.has(Hidden) || entity.has(IsMask)) return;
+ if (computed.visibility[entity.id()] === 0) return;
+
+ bounds.minX = Math.min(bounds.minX, worldBounds.minX[entity.id()]!);
+ bounds.minY = Math.min(bounds.minY, worldBounds.minY[entity.id()]!);
+ bounds.maxX = Math.max(bounds.maxX, worldBounds.maxX[entity.id()]!);
+ bounds.maxY = Math.max(bounds.maxY, worldBounds.maxY[entity.id()]!);
+
+ if (entity.has(ClipsContent)) return;
+ for (const child of world.query(Or(Geometry, Group), ChildOf(entity), Not(Deleted))) {
+ measure(child, bounds);
+ }
+ };
+
+ // Warm-up tick: group bounds aggregate from their children's local
+ // matrices, which don't exist until the first transform walk has visited
+ // them — measuring on that walk would union stale boxes.
+ seek(config.frames[0]!);
+ motionSystem(world);
+ transformSystem(world);
+
+ // Measure pass: run motion + transforms over every requested frame with an
+ // identity camera and union the bounds, so an animated node stays fully
+ // visible in a fixed-size output across all frames.
+ const bounds: AABB = { minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity };
+ for (const frame of config.frames) {
+ seek(frame);
+ motionSystem(world);
+ transformSystem(world);
+ measure(root, bounds);
+ }
+ assert(Number.isFinite(bounds.minX), 'Entity is not visible at any of the requested frames');
+
+ const boundsWidth = Math.max(1, bounds.maxX - bounds.minX);
+ const boundsHeight = Math.max(1, bounds.maxY - bounds.minY);
+
+ /** Re-target the output height; callers that lay frames out do this once measured. */
+ const resize = (height?: number) => {
+ const scale = height ? height / boundsHeight : 1;
+ offscreenCanvas.width = Math.max(1, Math.round(boundsWidth * scale));
+ offscreenCanvas.height = Math.max(1, Math.round(boundsHeight * scale));
+ world.set(RenderSurface, { resolution: scale });
+ };
+
+ resize(config.resolution);
+ // Shift the measured box onto the canvas so the node is centered in view;
+ // the camera translation is scaled by the surface resolution at render time.
+ world.set(Camera, { e: -bounds.minX, f: -bounds.minY });
+
+ let canceled = false;
+ const cancel = () => (canceled = true);
+
+ const render = async (): Promise<ImageExportResult> => {
+ try {
+ const images: CapturedImage[] = [];
+
+ for (const frame of config.frames) {
+ if (canceled) {
+ return { type: 'canceled' };
+ }
+
+ seek(frame);
+ await resolverSystem(world);
+ motionSystem(world);
+ transformSystem(world);
+ renderSystem(world);
+
+ const stampFrame = startFrame + frame;
+ images.push({
+ base64: await toBase64Png(offscreenCanvas),
+ timecode: formatTimecode(framesToSeconds(stampFrame, sourceFrameRate), sourceFrameRate),
+ });
+ }
+
+ return { type: 'success', data: images };
+ } catch (e) {
+ return {
+ type: 'error',
+ error: e instanceof Error ? e : new Error('Unknown error'),
+ };
+ } finally {
+ // Free the graphs + hosts realized above (this offline world is
+ // discarded; realized HtmlHosts otherwise leak a canvas on
+ // document.body), then release decoders and the world slot itself
+ // (koota caps live worlds at 16).
+ mounts?.disposeAll();
+ disposeDecoders(world, getDocument(world));
+ world.destroy();
+ }
+ };
+
+ return {
+ render,
+ cancel,
+ resize,
+ bounds: { width: boundsWidth, height: boundsHeight },
+ };
+}
+
+async function toBase64Png(canvas: OffscreenCanvas): Promise<string> {
+ const blob = await canvas.convertToBlob({ type: 'image/png' });
+ const dataUrl = await new Promise<string>((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(reader.result as string);
+ reader.onerror = () => reject(reader.error);
+ reader.readAsDataURL(blob);
+ });
+ return dataUrl.split(',')[1] ?? '';
+}
diff --git a/packages/encoder/src/index.ts b/packages/encoder/src/index.ts
new file mode 100644
index 00000000..87d54003
--- /dev/null
+++ b/packages/encoder/src/index.ts
@@ -0,0 +1,11 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export * from './buffer';
+export * from './encoder';
+export * from './format';
+export * from './image-encoder';
+export * from './interfaces';
+export * from './types';
+export * from './utils';
diff --git a/packages/encoder/src/interfaces.ts b/packages/encoder/src/interfaces.ts
new file mode 100644
index 00000000..57a539e6
--- /dev/null
+++ b/packages/encoder/src/interfaces.ts
@@ -0,0 +1,155 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { VideoCodec, AudioCodec } from 'mediabunny';
+import type { Entity, World } from 'koota';
+import type { ContainerFormat, EncoderProgress, RealizedMounts, WritableFileTarget } from './types';
+
+export interface AudioConfig {
+ /**
+ * Enable audio encoding
+ * @default true
+ */
+ enabled?: boolean;
+
+ /**
+ * A floating point number indicating the audio context's sample rate, in samples per second.
+ * @default 48000
+ */
+ sampleRate?: number;
+
+ /**
+ * Defines the number of channels of the composed audio
+ * @default 2
+ */
+ numberOfChannels?: number;
+
+ /**
+ * Defines the bitrate at which the audio should be rendered at
+ * @default 128e3
+ */
+ bitrate?: number;
+
+ /**
+ * Defines the codec to use for the audio
+ * @default 'aac'
+ */
+ codec?: AudioCodec;
+}
+
+export interface VideoConfig {
+ /**
+ * Defines the codec to use for the video
+ * @default 'avc'
+ */
+ codec?: VideoCodec;
+
+ /**
+ * The full codec string as specified in the WebCodecs API Codec Registry.
+ */
+ fullCodecString?: string;
+
+ /**
+ * Enable video encoding
+ * @default true
+ */
+ enabled?: boolean;
+
+ /**
+ * Defines the bitrate at which the video should be rendered at
+ * @default 10e6
+ */
+ bitrate?: number;
+
+ /**
+ * Defines the fps at which the composition will be rendered
+ * @default 30
+ */
+ fps?: number;
+
+ /**
+ * Resolution of the composition
+ * @example
+ * 1080 for 1080p
+ * 1440 for 1440p
+ * 2160 for 4k
+ * @default 1080
+ */
+ resolution?: number;
+}
+
+export interface EncoderConfig {
+ /**
+ * Video encoding configuration
+ */
+ video?: VideoConfig;
+
+ /**
+ * Audio encoding configuration
+ */
+ audio?: AudioConfig;
+
+ /**
+ * Defines the output format of the encoded file
+ * @default 'mp4'
+ */
+ format?: ContainerFormat;
+
+ /**
+ * Defines the range of the composition to encode
+ */
+ trim?: {
+ /**
+ * Defines the time when the encoding should end
+ * @default scene.duration
+ */
+ end?: number;
+ };
+
+ /**
+ * Defines the target file to encode to.
+ * When a FileSystemFileHandle (or any WritableFileTarget) is provided, the
+ * output is streamed to the file. When omitted, the output is buffered in
+ * memory and returned as a Blob in the ExportResult.
+ */
+ target?: FileSystemFileHandle | WritableFileTarget;
+
+ /**
+ * Defines the progress callback
+ */
+ onProgress?: (progress: EncoderProgress) => void;
+
+ /**
+ * Defines the source scene to encode
+ */
+ scene: Entity;
+
+ /**
+ * Re-executes the source world's mounts into the offline world so their
+ * reactive graphs and runtime hosts (surface/html) render during capture.
+ * Mount realization needs the host app's module loader, so it is injected.
+ */
+ realizeMounts?: (world: World) => Promise<RealizedMounts | null>;
+
+ /**
+ * Metadata comment tag written into the container (e.g. an app version).
+ */
+ comment?: string;
+}
+
+export interface ImageEncoderConfig {
+ /**
+ * The source entity to capture
+ */
+ entity: Entity;
+
+ /** Frames to capture, relative to the entity's first visible frame. */
+ frames: number[];
+
+ /** Target output height in px (default: the entity's native size). */
+ resolution?: number;
+
+ /** See EncoderConfig.realizeMounts. */
+ realizeMounts?: (world: World) => Promise<RealizedMounts | null>;
+}
diff --git a/packages/encoder/src/types.ts b/packages/encoder/src/types.ts
new file mode 100644
index 00000000..e584cd8b
--- /dev/null
+++ b/packages/encoder/src/types.ts
@@ -0,0 +1,36 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { StreamTargetChunk } from 'mediabunny';
+
+// The container format is part of the persisted document model (the
+// ExportSettings trait), so the runtime owns the type.
+export type { ContainerFormat } from '@diffusionstudio/runtime';
+
+// Minimal FileSystemFileHandle-shaped target
+export interface WritableFileTarget {
+ createWritable(): Promise<WritableStream<StreamTargetChunk>>;
+}
+
+export type EncoderProgress = {
+ total: number;
+ progress: number;
+ remaining: Date;
+};
+
+export type ExportResult =
+ | {
+ type: 'success';
+ data: Blob | undefined;
+ }
+ | {
+ type: 'canceled';
+ }
+ | {
+ type: 'error';
+ error: Error;
+ };
+
+/** Mount graphs re-executed into an offline world (see EncoderConfig.realizeMounts). */
+export type RealizedMounts = { disposeAll(): void };
diff --git a/packages/encoder/src/utils.ts b/packages/encoder/src/utils.ts
new file mode 100644
index 00000000..99496a5a
--- /dev/null
+++ b/packages/encoder/src/utils.ts
@@ -0,0 +1,22 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Helper for creating the render event detail
+ */
+export function createRenderEventDetail(progress: number, total: number, startTime: number) {
+ const duration = performance.now() - startTime;
+ const time = (duration / gte1(progress)) * (total - progress);
+ const remaining = new Date(time);
+
+ return { remaining, progress, total };
+}
+
+/**
+ * Helper for making sure a number is greater than 1
+ */
+function gte1(num: number): number {
+ if (num < 1) return 1;
+ return num;
+}
diff --git a/packages/encoder/tsconfig.json b/packages/encoder/tsconfig.json
new file mode 100644
index 00000000..c150db11
--- /dev/null
+++ b/packages/encoder/tsconfig.json
@@ -0,0 +1,10 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo",
+ "erasableSyntaxOnly": false,
+ "lib": ["es2024", "DOM", "DOM.Iterable"],
+ "types": ["@webgpu/types", "@types/audioworklet"]
+ },
+ "include": ["src"]
+}
diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.md
index c0001200..f38d7721 100644
--- a/packages/runtime/MIGRATION.md
+++ b/packages/runtime/MIGRATION.md
@@ -38,6 +38,10 @@ Working doc for migrating the editor engine from bitecs (in `apps/web/src/compon
| Fonts | `src/fonts/{fixtures,utils}.ts` | WebFonts fixtures verbatim; `loadWebFont` uses the Fonts world trait, resolves the FontFaceSet via globalThis (worker-capable), guards on missing FontFace (headless), and drops db persistence (app wraps it). `getLocalFonts` + `restoreFonts` stay app-side |
| Support utils | `src/utils/{color,async}.ts`, `clamp` in math | parseColor/colorToHex/rgbToColor (colord dep), AsyncMutex/attempt |
| Playback system | `src/systems/playback.ts`, `src/utils/transition.ts` | Playhead advance, visibility/local-time recursion, decoder forwarding, transitions, solo muting, live mounts step; `resolveAudioBus` exported from here. See semantic notes |
+| Render system | `src/systems/render.ts`, `HitRegions` world trait | Full traversal port: paints (solid/gradient/media/waveform/shader/html/surface), shadows/strokes/effects, masks, ClipsContent, transitions, Generating pulse. No-op without a RenderSurface. See semantic notes |
+| System schedule | `src/systems/schedule.ts` | `runSystems(world)`: playback → motion → transform → render, shared by the realtime tick and the capture loop |
+| Encoders | `packages/encoder` (`@diffusionstudio/encoder`) | `engine/encode/*` extracted into its OWN package, not `src/capture/`: createEncoder + createImageEncoder as pure consumers of the runtime's public API. See semantic notes |
+| Timecode helpers | `src/utils/time.ts` | `formatTimecode`/`formatTimestamp` (from `utils/transcode.ts`; the image encoder stamps captures with them) |
| Seeds (types only, rest follows) | `src/utils/text.ts` (Token/Line/TokenOptions), `src/capture/format.ts` (ContainerFormat) | The remaining engine files merge into these on their slice |
Verification style used so far: `npx tsc --noEmit` in the package, plus throwaway `smoke-test.tmp.ts` in the package root run with `npm exec --yes tsx@latest -- smoke-test.tmp.ts`, deleted afterwards (no test infra in the package yet). koota caps live worlds at 16; call `world.destroy()` in test loops.
@@ -74,7 +78,11 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All
- DOM caveats, all lazy (nothing touches DOM at import time): HtmlHost/SurfaceHost create elements at construction (html-in-canvas is Chromium-flag-only anyway), ShaderHost needs WebGPU, `loadWebFont` no-ops without FontFace. tsconfig gained `DOM.Iterable` + `@webgpu/types`.
- `SequenceDecoder` iterates the structural `AssetDirectoryHandle` (entries with `getFile`) instead of concrete FileSystemFileHandle, so node hosts can feed frame directories.
- Playback: the bitecs top-level query `Not(ChildOf('*'))` became `ChildOf(document)`. Playhead advance, auto-stop, and visibility/localTime writes stay raw store writes (no change events, same as bitecs), and `AudioPlayback` is lazily added to playback roots on first tick. `resolveAudioBus` returns null when the world has no AudioEngine context (headless-without-audio; `AudioDecoder.playTo(null, ...)` already no-ops), and the context/offset reads guard the same way. The transition-cleanup `removeComponent(..., false)` became a plain `entity.remove(Transition)`; the app's history layer must treat Transition onRemove during a tick as system-driven, not undoable.
-- `systems/schedule.ts` is still a stub: the realtime tick order is playback → motion → transform → render (per `api/engine.ts`), and the helper composing it lands with the render/capture slices when all members exist.
+- Render: `world.hitRegions` became the `HitRegions` world trait. The render system pushes CALLBACK-LESS entries ({ kind: 'entity', id } for Interactive entities, { kind: 'hud', id: 'canvas', quad } for the stage in realtime); the bitecs handleGeometryInteraction/handleCanvasInteraction bindings are the app's: its input system clears the list each frame, pushes HUD controls with handlers attached, and maps callback-less targets to its default handlers. In headless worlds nothing pushes (Interactive is app-added, the canvas region is realtime-only), so the list cannot grow unbounded without an input system draining it.
+- Render's top-level query `Not(ChildOf('*'))` became `ChildOf(document)`, like playback's. `getScaledImageProps` reads the ScaleMode trait against the ScaleModeType enum. `drawRectPath`, `renderFills`, and `renderNode` stay exported (HUD and thumbnail code consume them app-side).
+- Encoders live in `@diffusionstudio/encoder` (deps: runtime, mediabunny, koota), which imports ONLY the runtime's public API — the boundary that motivated the extraction. `ContainerFormat` stays in the runtime (`src/capture/format.ts`) because the ExportSettings trait persists it; the encoder re-exports the type. Nothing inside the runtime may import the encoder (cycle), so encoder-driven utilities like `utils/captions.ts` stay app-side or move into the encoder package.
+- Encoder API changes vs `engine/encode`: `config.scene`/`config.entity` are Entities; `realizeMounts` is injected via config (mount realization needs the host app's module loader + @diffusionstudio/jsx); the hardcoded "Made with Diffusion v..." metadata tag became `config.comment`; the bitecs `Hierarchy(ChildOf)` two-pass rescale walks became `recomputeAllTimeRanges` (exported) over `propagateTimeRangeDown` from the document's top-level nodes; and both encoders now dispose decoders and `world.destroy()` their throwaway offline worlds in `render()`'s finally (koota caps live worlds at 16 — bitecs could just drop them). Offline-world wiring is world traits (`Mode`/`Assets` shared by reference/`Fonts` copied/`RenderSurface`/`AudioEngine`/`FrameRate`/`FramePromises`) instead of the context bag; playhead/Playback/AudioPlayback writes stay raw store writes, matching the systems.
+- The encoder package's tsconfig must mirror the runtime's (`DOM.Iterable`, `@webgpu/types`, plus `@types/audioworklet`): runtime exports point at TS sources, so consumers compile them with their own compiler settings.
- `switchActiveScene` only sets the ActiveScene world trait; timeline buffer clearing and index rebuild stay app-side. `frameSelection` calls it once (bitecs called it redundantly inside and outside the transaction).
- `insertAssetInTimeline` returns the created entity or null when no scene is active (no toast). Folder ops are sync and return the changed records (`deleteFolder` returns removed folder/asset ids) so the app can persist; `moveAssetsToFolder` updates records in place instead of re-fronting them like `saveAsset` did. Panel navigation state (current folder) stays app-side.
- Decoder/host handle traits (`src/traits/media.ts`) are AoS traits typed as opaque `object` until the decoders move in; `entity.get(VideoDecoder)` returns the instance directly.
@@ -93,11 +101,9 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All
## Remaining work (suggested order)
-1. **Capture** (`src/capture/`): `engine/encode/*` (encoder, image-encoder, buffer, format, utils). Uses Mode, FramePromises, RenderSurface. Fill `systems/schedule.ts` once render exists.
-2. **Render system**: `systems/render.ts` is runtime-capable (OffscreenCanvas) and needed for headless capture; text rendering and gradients are already in (`utils/text.ts`, `systems/gradients.ts`; `addStopsTo` stayed private to gradients).
-3. **Remaining utils**: captions, contact-sheet, audio-sync, transcode (audit each for DOM/app deps; DOM-touching ones stay app-side).
-4. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.
-5. **CLI**: point node capture at the runtime package.
+1. **Remaining utils**: captions (needs the encoder package, so it lands there or app-side), contact-sheet, audio-sync, transcode (timecode helpers already moved; audit the rest for DOM/app deps; DOM-touching ones stay app-side).
+2. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.
+3. **CLI**: point node capture at the runtime + encoder packages.
## Layout reminders
diff --git a/packages/runtime/src/capture/format.ts b/packages/runtime/src/capture/format.ts
index 41110876..9127dd0c 100644
--- a/packages/runtime/src/capture/format.ts
+++ b/packages/runtime/src/capture/format.ts
@@ -2,7 +2,8 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Container format for export. The rest of engine/encode/format.ts joins this
-// file when capture moves into the runtime.
+// Container format for export. Lives in the runtime (not the encoder
+// package) because the ExportSettings trait persists it; the encoder
+// imports it from here.
export type ContainerFormat = 'mp4' | 'webm' | 'ogg' | 'mov';
diff --git a/packages/runtime/src/capture/index.ts b/packages/runtime/src/capture/index.ts
index 68f561e5..af486a5b 100644
--- a/packages/runtime/src/capture/index.ts
+++ b/packages/runtime/src/capture/index.ts
@@ -1,2 +1,3 @@
-// Capture and encoding: encoder, buffer, image-encoder (move wholesale from engine/encode/).
+// Persisted capture model. The encoders themselves live in
+// @diffusionstudio/encoder; only what the document format needs stays here.
export * from './format';
diff --git a/packages/runtime/src/systems/index.ts b/packages/runtime/src/systems/index.ts
index ac687b3e..15f0ebb2 100644
--- a/packages/runtime/src/systems/index.ts
+++ b/packages/runtime/src/systems/index.ts
@@ -3,3 +3,5 @@ export * from './transform';
export * from './motion';
export * from './gradients';
export * from './playback';
+export * from './render';
+export * from './schedule';
diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.ts
new file mode 100644
index 00000000..aca6963e
--- /dev/null
+++ b/packages/runtime/src/systems/render.ts
@@ -0,0 +1,868 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Render system (was systems/render.ts): draws the document tree onto the
+// world's RenderSurface. Runs identically against the editor canvas and an
+// OffscreenCanvas during capture; without a surface it is a no-op. Hit
+// regions are pushed callback-less (see HitRegions); the app's input layer
+// attaches its handlers.
+
+import { Not, Or } from 'koota';
+import { cubicBezier } from 'animejs';
+
+import { store } from '../world/store';
+import {
+ COMPOSITE_OPERATIONS, EffectType, GeometryType, PaintType, ScaleModeType,
+ StrokeCap, StrokeJoin, TransitionType,
+} from '../constants';
+import {
+ ChildOf, Deleted, Hidden, Culled, Generating, Interactive, IsMask,
+ ClipsContent, Geometry, Group, Paint, Caption, ScaleMode, Shader,
+ Appearance, Effect, StrokeStyle, AssetId, Transition, MixedCornerRadius,
+ LocalTransform, WorldTransform, Computed, Cache,
+ HtmlHostHandle, SurfaceHostHandle,
+ Mode, Time, FrameRate, Camera, Background, RenderSurface, Assets,
+ HitRegions,
+} from '../traits';
+import { getDocument, getParentNode } from '../queries/hierarchy';
+import { colorToHex } from '../utils/color';
+import { renderText } from '../utils/text';
+import { getTransitionWindow } from '../utils/transition';
+import { createLinearGradient, createRadialGradient } from './gradients';
+import {
+ resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder,
+ resolveCaptionDecoder, resolveShaderHost, getAudioPeaks,
+} from '../media';
+
+import type { Entity, World } from 'koota';
+import type { Quad } from '../math/aabb';
+
+type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+
+const MISSING_ASSET_COLOR = '#5C2828';
+
+function getCtx(world: World): Ctx2D {
+ return world.get(RenderSurface)!.ctx!;
+}
+
+export function drawRectPath(world: World, entity: Entity): void {
+ const ctx = getCtx(world);
+ const computed = store(world, Computed);
+ const eid = entity.id();
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+
+ const hasMixed = entity.has(MixedCornerRadius);
+ let tl = hasMixed ? computed.cornerRadiusTopLeft[eid]! : computed.cornerRadius[eid]!;
+ let tr = hasMixed ? computed.cornerRadiusTopRight[eid]! : tl;
+ let br = hasMixed ? computed.cornerRadiusBottomRight[eid]! : tl;
+ let bl = hasMixed ? computed.cornerRadiusBottomLeft[eid]! : tl;
+
+ ctx.beginPath();
+
+ if (tl === 0 && tr === 0 && br === 0 && bl === 0) {
+ ctx.rect(0, 0, w, h);
+ } else if (tl === tr && tr === br && br === bl) {
+ ctx.roundRect(0, 0, w, h, tl);
+ } else {
+ // Clamp radii so adjacent corners don't exceed the edge length (CSS spec algorithm)
+ const scale = Math.min(
+ w / (tl + tr || 1),
+ h / (tr + br || 1),
+ w / (br + bl || 1),
+ h / (bl + tl || 1),
+ 1,
+ );
+ if (scale < 1) {
+ tl *= scale;
+ tr *= scale;
+ br *= scale;
+ bl *= scale;
+ }
+
+ ctx.moveTo(tl, 0);
+ ctx.lineTo(w - tr, 0);
+ if (tr > 0) ctx.arcTo(w, 0, w, tr, tr);
+ else ctx.lineTo(w, 0);
+ ctx.lineTo(w, h - br);
+ if (br > 0) ctx.arcTo(w, h, w - br, h, br);
+ else ctx.lineTo(w, h);
+ ctx.lineTo(bl, h);
+ if (bl > 0) ctx.arcTo(0, h, 0, h - bl, bl);
+ else ctx.lineTo(0, h);
+ ctx.lineTo(0, tl);
+ if (tl > 0) ctx.arcTo(0, 0, tl, 0, tl);
+ else ctx.lineTo(0, 0);
+ }
+
+ ctx.closePath();
+}
+
+function getScaledImageProps(
+ mode: number,
+ imgW: number,
+ imgH: number,
+ targetW: number,
+ targetH: number,
+): [dx: number, dy: number, sw: number, sh: number] {
+ if (mode === ScaleModeType.FILL) {
+ return [0, 0, targetW, targetH];
+ }
+ if (mode === ScaleModeType.FIT) {
+ const scale = Math.min(targetW / imgW, targetH / imgH);
+ const sw = imgW * scale;
+ const sh = imgH * scale;
+ return [(targetW - sw) / 2, (targetH - sh) / 2, sw, sh];
+ }
+ if (mode === ScaleModeType.COVER) {
+ const scale = Math.max(targetW / imgW, targetH / imgH);
+ const sw = imgW * scale;
+ const sh = imgH * scale;
+ return [(targetW - sw) / 2, (targetH - sh) / 2, sw, sh];
+ }
+ // ScaleModeType.NONE — original size
+ return [0, 0, imgW, imgH];
+}
+
+const EPSILON = 1e-4;
+
+/** Build a single CSS filter fragment from an effect sub-entity. Returns null if hidden or no-op. */
+function effectFilter(world: World, sub: Entity): string | null {
+ if (sub.has(Hidden)) return null;
+
+ const value = store(world, Computed).value[sub.id()]!;
+ const type = store(world, Effect).type[sub.id()] ?? 0;
+
+ if (type === EffectType.LAYER_BLUR) {
+ const clamped = Math.max(0, value);
+ return clamped > EPSILON ? `blur(${clamped}px)` : null;
+ }
+ if (type === EffectType.BRIGHTNESS) {
+ const clamped = Math.min(1, Math.max(0, value));
+ return Math.abs(clamped - 1) > EPSILON ? `brightness(${clamped})` : null;
+ }
+ if (type === EffectType.CONTRAST) {
+ const clamped = Math.min(1, Math.max(0, value));
+ return Math.abs(clamped - 1) > EPSILON ? `contrast(${clamped})` : null;
+ }
+ if (type === EffectType.GRAYSCALE) {
+ const clamped = Math.min(1, Math.max(0, value));
+ return clamped > EPSILON ? `grayscale(${clamped})` : null;
+ }
+ if (type === EffectType.HUE_ROTATION) {
+ return Math.abs(value) > EPSILON ? `hue-rotate(${value}deg)` : null;
+ }
+ if (type === EffectType.INVERT) {
+ const clamped = Math.min(1, Math.max(0, value));
+ return clamped > EPSILON ? `invert(${clamped})` : null;
+ }
+ if (type === EffectType.SATURATE) {
+ const clamped = Math.min(1, Math.max(0, value));
+ return Math.abs(clamped - 1) > EPSILON ? `saturate(${clamped})` : null;
+ }
+ if (type === EffectType.SEPIA) {
+ const clamped = Math.min(1, Math.max(0, value));
+ return clamped > EPSILON ? `sepia(${clamped})` : null;
+ }
+ return null;
+}
+
+/** CSS filter string from the entity's own blur plus effect sub-entities. */
+function buildEffects(world: World, entity: Entity): string | null {
+ const parts: string[] = [];
+
+ const blurVal = store(world, Computed).blur[entity.id()]!;
+ if (blurVal > EPSILON) {
+ parts.push(`blur(${blurVal}px)`);
+ }
+
+ const effects = store(world, Cache).effects[entity.id()] ?? [];
+ for (const effect of effects) {
+ const f = effectFilter(world, effect);
+ if (f) parts.push(f);
+ }
+
+ if (parts.length === 0) return null;
+
+ return parts.join(' ');
+}
+
+export function renderFills(world: World, entity: Entity): void {
+ const ctx = getCtx(world);
+ const computed = store(world, Computed);
+ const paintStore = store(world, Paint);
+ const appearance = store(world, Appearance);
+ const scaleMode = store(world, ScaleMode);
+ const eid = entity.id();
+ const fills = store(world, Cache).fills[eid] ?? [];
+
+ for (let index = 0; index < fills.length; index++) {
+ const fill = fills[index]!;
+ if (fill.has(Hidden) || shaderConsumesFill(world, entity, fills, index)) continue;
+ const fid = fill.id();
+ const savedAlpha = ctx.globalAlpha;
+ const savedCO = ctx.globalCompositeOperation;
+ const bi = appearance.blendMode[fid] ?? 0;
+
+ if (bi !== 0) {
+ ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[bi]!;
+ }
+
+ ctx.globalAlpha = savedAlpha * computed.opacity[fid]!;
+
+ const paint = paintStore.value[fid];
+ if (paint === PaintType.IMAGE) {
+ const decoder = resolveImageDecoder(world, fill)?.decoder;
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+ const bitmap = decoder?.getBitmap(w, h);
+
+ if (bitmap) {
+ ctx.save();
+ ctx.clip();
+
+ const mode = scaleMode.value[fid] ?? 0;
+ const [dx, dy, sw, sh] = getScaledImageProps(mode, bitmap.width, bitmap.height, w, h);
+ ctx.drawImage(bitmap, dx, dy, sw, sh);
+
+ ctx.restore();
+ } else if (decoder?.failed) {
+ ctx.fillStyle = MISSING_ASSET_COLOR;
+ ctx.fill();
+ }
+ } else if (paint === PaintType.VIDEO) {
+ const decoder = resolveVideoDecoder(world, fill);
+ const frame = decoder?.toBitmap();
+
+ if (frame) {
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+
+ ctx.save();
+ ctx.clip();
+
+ const mode = scaleMode.value[fid] ?? 0;
+ const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);
+ ctx.drawImage(frame, dx, dy, sw, sh);
+
+ ctx.restore();
+ } else if (decoder?.errored) {
+ ctx.fillStyle = MISSING_ASSET_COLOR;
+ ctx.fill();
+ }
+ } else if (paint === PaintType.SEQUENCE) {
+ const decoder = resolveSequenceDecoder(world, fill);
+ const frame = decoder?.toBitmap();
+
+ if (frame) {
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+
+ ctx.save();
+ ctx.clip();
+
+ const mode = scaleMode.value[fid] ?? 0;
+ const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);
+ ctx.drawImage(frame, dx, dy, sw, sh);
+
+ ctx.restore();
+ } else if (decoder?.errored) {
+ ctx.fillStyle = MISSING_ASSET_COLOR;
+ ctx.fill();
+ }
+ } else if (paint === PaintType.HTML) {
+ const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;
+
+ if (host) {
+ ctx.save();
+ ctx.clip();
+ host.draw(ctx, computed.width[eid]!, computed.height[eid]!);
+ ctx.restore();
+ }
+ } else if (paint === PaintType.SURFACE) {
+ const host = fill.has(SurfaceHostHandle) ? fill.get(SurfaceHostHandle) : null;
+
+ if (host) {
+ ctx.save();
+ ctx.clip();
+ host.draw(ctx, computed.width[eid]!, computed.height[eid]!);
+ ctx.restore();
+ }
+ } else if (paint === PaintType.SOLID) {
+ ctx.fillStyle = colorToHex(computed.color[fid]!);
+ ctx.fill();
+ } else if (paint === PaintType.LINEAR_GRADIENT) {
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+ ctx.fillStyle = createLinearGradient(world, fill, ctx, w, h);
+ ctx.fill();
+ } else if (paint === PaintType.RADIAL_GRADIENT) {
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+ ctx.fillStyle = createRadialGradient(world, fill, ctx, w, h);
+ ctx.fill();
+ } else if (paint === PaintType.WAVEFORM) {
+ renderWaveform(world, entity, fill);
+ } else if (paint === PaintType.SHADER) {
+ renderShaderFill(world, entity, fills, index);
+ }
+
+ ctx.globalCompositeOperation = savedCO;
+ ctx.globalAlpha = savedAlpha;
+ }
+}
+
+/** The current frame of a video/image paint, as a GPU-uploadable source. */
+function shaderSourceBitmap(
+ world: World,
+ fill: Entity,
+ w: number,
+ h: number,
+): { source: GPUCopyExternalImageSource; width: number; height: number } | null {
+ const paint = store(world, Paint).value[fill.id()];
+
+ if (paint === PaintType.IMAGE) {
+ const bitmap = resolveImageDecoder(world, fill)?.decoder?.getBitmap(w, h);
+ return bitmap ? { source: bitmap, width: bitmap.width, height: bitmap.height } : null;
+ }
+ if (paint === PaintType.VIDEO) {
+ const frame = resolveVideoDecoder(world, fill)?.toBitmap();
+ return frame ? { source: frame, width: frame.width, height: frame.height } : null;
+ }
+ return null;
+}
+
+/**
+ * Whether the fill at `index` is the input of a ready shader paint directly
+ * above it. Only the immediate neighbor counts (a hidden paint in between
+ * decouples the pair). Must mirror `renderShaderFill`'s input selection
+ * exactly — a consumed media paint that the shader then fails to draw would
+ * blank the element, so both sides check pipeline readiness and frame
+ * availability.
+ */
+function shaderConsumesFill(world: World, entity: Entity, fills: Entity[], index: number): boolean {
+ const paintStore = store(world, Paint);
+ const computed = store(world, Computed);
+ const fill = fills[index]!;
+ const paint = paintStore.value[fill.id()];
+ if (paint !== PaintType.IMAGE && paint !== PaintType.VIDEO) return false;
+
+ const next = fills[index + 1];
+ if (next === undefined || paintStore.value[next.id()] !== PaintType.SHADER) return false;
+ if (next.has(Hidden)) return false;
+ if (!resolveShaderHost(world, next)?.ready) return false;
+
+ const w = computed.width[entity.id()]!;
+ const h = computed.height[entity.id()]!;
+ return shaderSourceBitmap(world, fill, w, h) !== null;
+}
+
+/**
+ * Draws a shader paint: the media paint directly below it in the fill stack
+ * is sampled as the shader's `source` texture and its output lands in the
+ * parent's box in the media paint's place. Without a media paint below the
+ * shader runs procedurally over a transparent source; before the pipeline is
+ * ready it draws nothing and the media, if any, draws normally.
+ */
+function renderShaderFill(world: World, entity: Entity, fills: Entity[], index: number): void {
+ const ctx = getCtx(world);
+ const paintStore = store(world, Paint);
+ const computed = store(world, Computed);
+
+ const host = resolveShaderHost(world, fills[index]!);
+ if (!host?.ready) return;
+
+ const eid = entity.id();
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+
+ // A visible media paint directly below is the shader's input. Anything
+ // else (no fill below, a hidden one, a solid/gradient) runs the shader
+ // procedurally over a transparent source, stacking like a normal paint.
+ const media = fills[index - 1];
+ const isMedia = media !== undefined
+ && !media.has(Hidden)
+ && (paintStore.value[media.id()] === PaintType.IMAGE || paintStore.value[media.id()] === PaintType.VIDEO);
+ let input: ReturnType<typeof shaderSourceBitmap> = null;
+ if (isMedia) {
+ input = shaderSourceBitmap(world, media, w, h);
+ if (!input) return;
+ }
+
+ const fit = input
+ ? getScaledImageProps(store(world, ScaleMode).value[media!.id()] ?? 0, input.width, input.height, w, h)
+ : [0, 0, w, h] as [number, number, number, number];
+ const fps = world.get(FrameRate)?.value ?? 30;
+ const time = (computed.localTime[eid] ?? 0) / fps;
+
+ ctx.save();
+ ctx.clip();
+ host.draw(ctx, w, h, input?.source ?? null, input?.width ?? 1, input?.height ?? 1, fit, time, store(world, Shader).uniforms[fills[index]!.id()] ?? null);
+ ctx.restore();
+}
+
+function renderShadows(world: World, entity: Entity): void {
+ const ctx = getCtx(world);
+ const computed = store(world, Computed);
+
+ const shadows = store(world, Cache).shadows[entity.id()];
+ if (!shadows) return;
+
+ ctx.save();
+ const savedAlpha = ctx.globalAlpha;
+
+ // ctx.shadowBlur/OffsetX/OffsetY are in device-pixel space and are not
+ // affected by the current transform, so scale them up to match the
+ // content transform (camera * resolution).
+ const camera = world.get(Camera);
+ const resolution = world.get(RenderSurface)?.resolution ?? 1;
+ const shadowScale = (camera?.a ?? 1) * resolution;
+
+ for (const shadow of shadows) {
+ if (shadow.has(Hidden)) continue;
+ const sid = shadow.id();
+ const color = colorToHex(computed.color[sid]!);
+ ctx.shadowColor = color;
+ ctx.fillStyle = color;
+ ctx.globalAlpha = savedAlpha * computed.opacity[sid]!;
+ ctx.shadowBlur = computed.blur[sid]! * shadowScale;
+ ctx.shadowOffsetX = computed.offsetX[sid]! * shadowScale;
+ ctx.shadowOffsetY = computed.offsetY[sid]! * shadowScale;
+ ctx.fill();
+ }
+
+ ctx.restore();
+}
+
+function renderStrokes(world: World, entity: Entity): void {
+ const ctx = getCtx(world);
+ const eid = entity.id();
+ const strokes = store(world, Cache).strokes[eid];
+ if (!strokes) return;
+
+ const strokeStyle = store(world, StrokeStyle);
+ const computed = store(world, Computed);
+ const appearance = store(world, Appearance);
+ const paintStore = store(world, Paint);
+
+ ctx.lineWidth = computed.strokeWidth[eid]!;
+ ctx.lineJoin = StrokeJoin[strokeStyle.join[eid] ?? 0]!.toLocaleLowerCase() as CanvasLineJoin;
+ ctx.lineCap = StrokeCap[strokeStyle.cap[eid] ?? 0]!.toLocaleLowerCase() as CanvasLineCap;
+ ctx.miterLimit = strokeStyle.miterLimit[eid] ?? 3;
+
+ for (const stroke of strokes) {
+ if (stroke.has(Hidden)) continue;
+ const sid = stroke.id();
+ const savedAlpha = ctx.globalAlpha;
+ const savedCO = ctx.globalCompositeOperation;
+ const bi = appearance.blendMode[sid] ?? 0;
+
+ if (bi !== 0) {
+ ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[bi]!;
+ }
+
+ ctx.globalAlpha = savedAlpha * computed.opacity[sid]!;
+
+ const paintType = paintStore.value[sid];
+ if (paintType === PaintType.LINEAR_GRADIENT) {
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+ ctx.strokeStyle = createLinearGradient(world, stroke, ctx, w, h);
+ } else if (paintType === PaintType.RADIAL_GRADIENT) {
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+ ctx.strokeStyle = createRadialGradient(world, stroke, ctx, w, h);
+ } else {
+ ctx.strokeStyle = colorToHex(computed.color[sid]!);
+ }
+ ctx.stroke();
+
+ ctx.globalCompositeOperation = savedCO;
+ ctx.globalAlpha = savedAlpha;
+ }
+}
+
+/**
+ * Pulse animation for entities with the Generating tag.
+ *
+ * Easing: cubic-bezier(0.52, 0.18, 0.56, 0.88)
+ * Duration: 0.6s per half-cycle (alternating)
+ * Delay: 0.2s before the first transition
+ * Full cycle: 0.2 delay + 0.6 forward + 0.6 reverse = 1.4s
+ * repeats ∞ for the entire gen phase (~4.5s)
+ *
+ * Pulses between --background (#1c1c1c) and --secondary (#292929).
+ */
+const generatingEase = cubicBezier(0.52, 0.18, 0.56, 0.88);
+
+const GEN_DELAY = 200; // ms
+const GEN_DURATION = 600; // ms per half-cycle
+const GEN_CYCLE = GEN_DURATION * 2; // full forward+reverse cycle
+
+// Colors: #1c1c1c → rgb(28,28,28) and #292929 → rgb(41,41,41)
+const GEN_FROM_R = 28, GEN_FROM_G = 28, GEN_FROM_B = 28;
+const GEN_TO_R = 41, GEN_TO_G = 41, GEN_TO_B = 41;
+
+function renderGenerating(world: World, entity: Entity): void {
+ const ctx = getCtx(world);
+ if (!entity.has(Generating)) return;
+
+ const t = world.get(Time)?.now ?? 0;
+
+ // Before initial delay → show base color
+ const elapsed = t % (GEN_DELAY + GEN_CYCLE);
+ let factor = 0;
+
+ if (elapsed >= GEN_DELAY) {
+ const cycleT = elapsed - GEN_DELAY;
+ const half = cycleT % GEN_DURATION;
+ const progress = half / GEN_DURATION;
+ const eased = generatingEase(progress);
+
+ // Alternate direction: first half forward, second half reverse
+ const inReverse = cycleT >= GEN_DURATION;
+ factor = inReverse ? 1 - eased : eased;
+ }
+
+ const r = Math.round(GEN_FROM_R + (GEN_TO_R - GEN_FROM_R) * factor);
+ const g = Math.round(GEN_FROM_G + (GEN_TO_G - GEN_FROM_G) * factor);
+ const b = Math.round(GEN_FROM_B + (GEN_TO_B - GEN_FROM_B) * factor);
+
+ ctx.fillStyle = `rgb(${r},${g},${b})`;
+ ctx.fill();
+}
+
+// ── WAVEFORM paint ─────────────────────────────────────
+//
+// Renders an audio asset's pre-computed peaks as a bar chart inside the
+// parent geometry's bounds. Sourced from the paint's own AssetId — the paint
+// carries its asset reference, exactly like IMAGE/VIDEO/SEQUENCE paints.
+
+const WAVEFORM_BAR_WIDTH = 6;
+const WAVEFORM_BAR_GAP = 6;
+const WAVEFORM_BAR_RADIUS = WAVEFORM_BAR_WIDTH / 2;
+const WAVEFORM_MIN_BAR_HEIGHT = 4;
+const WAVEFORM_PADDING = 12;
+const WAVEFORM_BG_COLOR = '#202020';
+const WAVEFORM_BG_RADIUS = 12;
+
+function renderWaveform(world: World, entity: Entity, fill: Entity): void {
+ const ctx = getCtx(world);
+ const computed = store(world, Computed);
+
+ const assetId = store(world, AssetId).value[fill.id()];
+ if (!assetId) return;
+
+ const asset = world.get(Assets)?.get(assetId);
+ // Both AUDIO and VIDEO assets carry pre-computed peaks (video has audio).
+ if (asset?.type !== 'AUDIO' && asset?.type !== 'VIDEO') return;
+
+ const peaks = getAudioPeaks(asset);
+ if (!peaks || peaks.length === 0) return;
+
+ const w = computed.width[entity.id()]!;
+ const h = computed.height[entity.id()]!;
+
+ ctx.save();
+ ctx.clip();
+
+ // Background
+ ctx.fillStyle = WAVEFORM_BG_COLOR;
+ ctx.beginPath();
+ ctx.roundRect(0, 0, w, h, WAVEFORM_BG_RADIUS);
+ ctx.fill();
+
+ // Bars
+ const step = WAVEFORM_BAR_WIDTH + WAVEFORM_BAR_GAP;
+ const availableWidth = w - WAVEFORM_PADDING * 2;
+ const barCount = Math.floor(availableWidth / step);
+ const maxBarHeight = h - WAVEFORM_PADDING * 2;
+ if (barCount <= 0 || maxBarHeight <= 0) {
+ ctx.restore();
+ return;
+ }
+
+ const startX = WAVEFORM_PADDING + (availableWidth - barCount * step + WAVEFORM_BAR_GAP) / 2;
+
+ ctx.fillStyle = '#ffffff';
+
+ for (let i = 0; i < barCount; i++) {
+ const peakIndex = Math.floor((i / barCount) * peaks.length);
+ const value = (peaks[peakIndex] ?? 0) / 255;
+ const barHeight = Math.max(value * maxBarHeight, WAVEFORM_MIN_BAR_HEIGHT);
+ const x = startX + i * step;
+ const y = (h - barHeight) / 2;
+
+ ctx.beginPath();
+ ctx.roundRect(x, y, WAVEFORM_BAR_WIDTH, barHeight, WAVEFORM_BAR_RADIUS);
+ ctx.fill();
+ }
+
+ ctx.restore();
+}
+
+function renderShapeNode(world: World, entity: Entity): void {
+ drawRectPath(world, entity);
+ renderShadows(world, entity);
+ renderFills(world, entity);
+ renderGenerating(world, entity);
+ renderStrokes(world, entity);
+}
+
+function renderTextNode(world: World, entity: Entity): void {
+ renderText(world, entity);
+}
+
+function renderCaptionNode(world: World, entity: Entity): void {
+ resolveCaptionDecoder(world, entity)?.draw(world, entity);
+}
+
+// ── Transition rendering ─────────────────────────────────────
+
+function renderTransition(world: World, scene: Entity, left: Entity): void {
+ const ctx = getCtx(world);
+ const computed = store(world, Computed);
+
+ const currentTime = computed.localTime[scene.id()]!;
+
+ const children = store(world, Cache).children[scene.id()] ?? [];
+ const right = children.find(sibling => computed.start[sibling.id()] === computed.end[left.id()]);
+ if (!right) return;
+
+ const win = getTransitionWindow(world, left, right);
+
+ if (currentTime < win.start || currentTime >= win.end) return;
+
+ // we are transitioning
+ const duration = win.end - win.start;
+ const completion = (currentTime - win.start) / duration;
+
+ const type = store(world, Transition).type[left.id()] ?? TransitionType.DISSOLVE;
+
+ const parent = getParentNode(left);
+ if (parent === null) return;
+ const width = computed.width[parent.id()]!;
+ const height = computed.height[parent.id()]!;
+
+ switch (type) {
+ case TransitionType.SLIDE_FROM_RIGHT: {
+ renderNode(world, left);
+ ctx.save();
+ ctx.translate(((1 - completion) ** 2 * width) | 0, 0);
+ renderNode(world, right);
+ ctx.restore();
+ break;
+ }
+ case TransitionType.SLIDE_FROM_LEFT: {
+ renderNode(world, left);
+ ctx.save();
+ ctx.translate(((1 - completion) ** 2 * width * -1) | 0, 0);
+ renderNode(world, right);
+ ctx.restore();
+ break;
+ }
+ case TransitionType.FADE_TO_BLACK: {
+ if (completion < 0.5) {
+ renderNode(world, left);
+ } else {
+ renderNode(world, right);
+ }
+ ctx.save();
+ ctx.beginPath();
+ ctx.rect(0, 0, width, height);
+ ctx.closePath();
+ ctx.fillStyle = '#000000';
+ ctx.globalAlpha = completion < 0.5 ? 2 * completion : 2 * (1 - completion);
+ ctx.fill();
+ ctx.restore();
+ break;
+ }
+ case TransitionType.FADE_TO_WHITE: {
+ if (completion < 0.5) {
+ renderNode(world, left);
+ } else {
+ renderNode(world, right);
+ }
+ ctx.save();
+ ctx.beginPath();
+ ctx.rect(0, 0, width, height);
+ ctx.closePath();
+ ctx.fillStyle = '#FFFFFF';
+ ctx.globalAlpha = completion < 0.5 ? 2 * completion : 2 * (1 - completion);
+ ctx.fill();
+ ctx.restore();
+ break;
+ }
+ default: {
+ // Dissolve (default)
+ renderNode(world, left);
+ ctx.save();
+ ctx.globalAlpha = completion;
+ renderNode(world, right);
+ ctx.restore();
+ break;
+ }
+ }
+
+ // Mark both partners as already drawn this frame so the parent's
+ // children loop skips its plain renderNode pass for them.
+ computed.visibility[left.id()] = 0;
+ computed.visibility[right.id()] = 0;
+}
+
+export function renderNode(world: World, entity: Entity): void {
+ const ctx = getCtx(world);
+ const computed = store(world, Computed);
+ const eid = entity.id();
+
+ if (computed.visibility[eid] === 0 || entity.has(Culled)) return;
+
+ if (entity.has(Interactive)) {
+ world.get(HitRegions)?.list.push({
+ target: { kind: 'entity', id: entity },
+ });
+ }
+
+ if (entity.has(IsMask) || entity.has(Hidden)) return;
+
+ ctx.save();
+
+ const local = store(world, LocalTransform);
+ ctx.transform(
+ local.a[eid]!,
+ local.b[eid]!,
+ local.c[eid]!,
+ local.d[eid]!,
+ local.e[eid]!,
+ local.f[eid]!,
+ );
+
+ const worldTransform = store(world, WorldTransform);
+ for (const mask of store(world, Cache).masks[eid] ?? []) {
+ if (computed.visibility[mask.id()] === 0) continue;
+ ctx.save();
+ ctx.setTransform(
+ worldTransform.a[mask.id()]!,
+ worldTransform.b[mask.id()]!,
+ worldTransform.c[mask.id()]!,
+ worldTransform.d[mask.id()]!,
+ worldTransform.e[mask.id()]!,
+ worldTransform.f[mask.id()]!,
+ );
+ drawRectPath(world, mask);
+ ctx.restore();
+ ctx.clip();
+ }
+
+ // Appearance
+ ctx.globalAlpha *= computed.opacity[eid]!;
+ const bi = store(world, Appearance).blendMode[eid] ?? 0;
+ if (bi !== 0) ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[bi]!;
+
+ const effects = buildEffects(world, entity);
+ let initialFilter = 'none';
+
+ if (effects !== null) {
+ initialFilter = ctx.filter;
+ ctx.filter = effects;
+ }
+
+
+ if (entity.has(Caption)) {
+ renderCaptionNode(world, entity);
+ } else if (store(world, Geometry).value[eid] === GeometryType.TEXT) {
+ renderTextNode(world, entity);
+ } else if (store(world, Geometry).value[eid] === GeometryType.RECT) {
+ renderShapeNode(world, entity);
+ }
+
+ // Clip and render children
+ const children = store(world, Cache).children[eid] ?? [];
+ if (children.length) {
+ if (entity.has(ClipsContent)) {
+ ctx.save();
+ ctx.clip();
+ }
+
+ for (const child of children) {
+ // Edge case: Child with transition
+ if (child.has(Transition)) {
+ renderTransition(world, entity, child);
+ // Note: we are not breaking here since the transition handler will hide/unhide the children
+ }
+
+ renderNode(world, child);
+ }
+
+ if (entity.has(ClipsContent)) {
+ ctx.restore();
+ }
+ }
+
+ // Reset filter after drawing
+ if (initialFilter !== 'none') {
+ ctx.filter = initialFilter;
+ }
+
+ ctx.restore();
+}
+
+/**
+ * Render system entry point. Call after transformSystem.
+ *
+ * Reads camera, background, and canvas size from world state and applies
+ * DPR * Camera as the base canvas transform before drawing top-level nodes.
+ * Without a render surface (headless world) this is a no-op.
+ */
+export function renderSystem(world: World): void {
+ const surface = world.get(RenderSurface);
+ const ctx = surface?.ctx;
+ const canvas = surface?.canvas;
+ if (!ctx || !canvas) return;
+
+ const cw = canvas.width;
+ const ch = canvas.height;
+
+ // Clear + background (identity transform for full-canvas clear)
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.clearRect(0, 0, cw, ch);
+
+ // The stage background is a preview-only affordance; offline encoding
+ // renders just the scene onto a transparent canvas (the scene paints its
+ // own fill if it has one).
+ if (world.get(Mode)?.value === 'realtime') {
+ ctx.fillStyle = colorToHex(world.get(Background)?.value ?? 0);
+ ctx.fillRect(0, 0, cw, ch);
+ world.get(HitRegions)?.list.push({
+ target: { kind: 'hud', id: 'canvas', quad: getCanvasQuad(cw, ch) },
+ });
+ }
+
+ // Apply camera transform: DPR * Camera
+ const camera = world.get(Camera) ?? { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 };
+ const resolution = surface.resolution;
+ ctx.setTransform(
+ camera.a * resolution,
+ camera.b * resolution,
+ camera.c * resolution,
+ camera.d * resolution,
+ camera.e * resolution,
+ camera.f * resolution,
+ );
+
+ // Render top-level nodes.
+ const document = getDocument(world);
+ for (const entity of world.query(Or(Geometry, Group), ChildOf(document), Not(Deleted), Not(Culled))) {
+ renderNode(world, entity);
+ }
+}
+
+function getCanvasQuad(width: number, height: number): Quad {
+ return [
+ { x: 0, y: 0 },
+ { x: width, y: 0 },
+ { x: width, y: height },
+ { x: 0, y: height },
+ ];
+}
diff --git a/packages/runtime/src/systems/schedule.ts b/packages/runtime/src/systems/schedule.ts
index 1f1213f5..10929567 100644
--- a/packages/runtime/src/systems/schedule.ts
+++ b/packages/runtime/src/systems/schedule.ts
@@ -1 +1,22 @@
-// System ordering for realtime vs capture mode.
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Deterministic system order for one frame, shared by the realtime tick and
+// the capture loop (which sets Time/playheads explicitly, runs this, then
+// awaits FramePromises before sampling). The app's tick wraps this with its
+// browser-only systems (input before, HUD/dom sync after).
+
+import { playbackSystem } from './playback';
+import { motionSystem } from './motion';
+import { transformSystem } from './transform';
+import { renderSystem } from './render';
+
+import type { World } from 'koota';
+
+export function runSystems(world: World): void {
+ playbackSystem(world);
+ motionSystem(world);
+ transformSystem(world);
+ renderSystem(world);
+}
diff --git a/packages/runtime/src/traits/world.ts b/packages/runtime/src/traits/world.ts
index 45d1c63c..01c0e943 100644
--- a/packages/runtime/src/traits/world.ts
+++ b/packages/runtime/src/traits/world.ts
@@ -8,6 +8,7 @@ import { createLiveMounts } from '../utils/live-mounts';
import type { FontSource } from '../fonts/types';
import type { Asset, Folder } from '../assets/types';
+import type { Quad } from '../math/aabb';
// Singleton state attached to the world itself (world.get/world.set).
// Replaces the bitecs createWorld({...}) context bag. Only headless runtime
@@ -84,3 +85,26 @@ export const Mounts = trait(() => createLiveMounts());
export const FramePromises = trait({
list: () => null as (Promise<unknown> | null)[] | null,
});
+
+export type EntityTarget = {
+ kind: 'entity';
+ id: Entity;
+};
+
+export type HudTarget = {
+ kind: 'hud';
+ id: string;
+ quad: Quad;
+};
+
+// Paint-order hit regions collected during a render pass (topmost last). The
+// render system pushes callback-less entries for Interactive entities and the
+// stage canvas; the app's input system clears the list each frame, pushes its
+// HUD controls with handlers attached, and maps callback-less targets to its
+// default interaction handlers.
+export type HitRegion = {
+ target: EntityTarget | HudTarget;
+ callback?: (world: unknown, event: unknown) => void;
+};
+
+export const HitRegions = trait({ list: () => [] as HitRegion[] });
diff --git a/packages/runtime/src/utils/time.ts b/packages/runtime/src/utils/time.ts
index 41bc5ad2..ebc6698e 100644
--- a/packages/runtime/src/utils/time.ts
+++ b/packages/runtime/src/utils/time.ts
@@ -26,6 +26,36 @@ export function framesToSeconds(frames: number = 0, fps: number = 30) {
return snapToMs(frames / fps);
}
+function splitTimecode(seconds: number, frameRate: number): [hours: number, minutes: number, seconds: number, frames: number] {
+ const fps = Math.max(1, Math.round(frameRate));
+ const totalFrames = Math.round(seconds * fps);
+ const totalSeconds = Math.floor(totalFrames / fps);
+ return [
+ Math.floor(totalSeconds / 3600),
+ Math.floor((totalSeconds % 3600) / 60),
+ totalSeconds % 60,
+ totalFrames % fps,
+ ];
+}
+
+/** `HH:MM:SS:FF`, fixed width so ruler ticks line up. */
+export function formatTimestamp(seconds: number, frameRate: number): string {
+ return splitTimecode(seconds, frameRate).map((v) => String(v).padStart(2, '0')).join(':');
+}
+
+/**
+ * The same timecode with its zero segments dropped, for labels and filenames:
+ * `08s10f`, `01m05s`, `0f`. Each segment carries its unit, so nothing is
+ * ambiguous once the empty ones are gone.
+ */
+export function formatTimecode(seconds: number, frameRate: number): string {
+ const units = ['h', 'm', 's', 'f'];
+ const stamp = splitTimecode(seconds, frameRate)
+ .map((value, i) => (value === 0 ? '' : `${String(value).padStart(2, '0')}${units[i]}`))
+ .join('');
+ return stamp || '0f';
+}
+
/**
* Computes the trim start frame (local) for a given entity,
* based on the specified start frame (global) and playback rate.
diff --git a/packages/runtime/src/world/create-world.ts b/packages/runtime/src/world/create-world.ts
index 01373fdb..f08db4b8 100644
--- a/packages/runtime/src/world/create-world.ts
+++ b/packages/runtime/src/world/create-world.ts
@@ -22,6 +22,7 @@ import {
AssetIds,
Mounts,
FramePromises,
+ HitRegions,
} from '../traits/world';
/**
@@ -47,6 +48,7 @@ export function createRuntimeWorld(projectId: string) {
AssetIds,
Mounts,
FramePromises,
+ HitRegions,
);
// The document root all rendered entities hang off (see DocumentRoot).