The VS Code of video editing, built for agents.
TypeScript
Update runtime package to include encoder system and enhance rendering capabilities
konstantin-paulus664eb897c8e4
diff --git a/package-lock.json b/package-lock.jsonindex 90d8070c..6a1de354 100644--- a/package-lock.json+++ b/package-lock.json@@ -593,6 +593,10 @@"resolved": "apps/desktop","link": true},+ "node_modules/@diffusionstudio/encoder": {+ "resolved": "packages/encoder",+ "link": true+ },"node_modules/@diffusionstudio/jsx": {"resolved": "packages/jsx","link": true@@ -12300,6 +12304,21 @@"url": "https://github.com/sponsors/colinhacks"}},+ "packages/encoder": {+ "name": "@diffusionstudio/encoder",+ "version": "0.1.0",+ "license": "MPL-2.0",+ "dependencies": {+ "@diffusionstudio/runtime": "*",+ "koota": "^0.6.6",+ "mediabunny": "^1.32.2"+ },+ "devDependencies": {+ "@types/audioworklet": "^0.0.92",+ "@webgpu/types": "^0.1.71",+ "typescript": "~5.9.3"+ }+ },"packages/jsx": {"name": "@diffusionstudio/jsx","version": "0.1.0",diff --git a/packages/encoder/package.json b/packages/encoder/package.jsonnew file mode 100644index 00000000..319639ac--- /dev/null+++ b/packages/encoder/package.json@@ -0,0 +1,29 @@+{+ "name": "@diffusionstudio/encoder",+ "version": "0.1.0",+ "description": "Offline video/audio/image encoding over @diffusionstudio/runtime worlds via mediabunny. Needs a browser-grade context (OffscreenCanvas, OfflineAudioContext, AudioWorklet, SharedArrayBuffer).",+ "license": "MPL-2.0",+ "type": "module",+ "sideEffects": false,+ "exports": {+ ".": "./src/index.ts"+ },+ "scripts": {+ "check": "tsc --noEmit"+ },+ "keywords": [+ "encoder",+ "video",+ "webcodecs"+ ],+ "dependencies": {+ "@diffusionstudio/runtime": "*",+ "koota": "^0.6.6",+ "mediabunny": "^1.32.2"+ },+ "devDependencies": {+ "@types/audioworklet": "^0.0.92",+ "@webgpu/types": "^0.1.71",+ "typescript": "~5.9.3"+ }+}diff --git a/packages/encoder/src/buffer.ts b/packages/encoder/src/buffer.tsnew file mode 100644index 00000000..72705ae7--- /dev/null+++ b/packages/encoder/src/buffer.ts@@ -0,0 +1,50 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { BufferTarget, StreamTarget, Target } from 'mediabunny';++import type { ContainerFormat, WritableFileTarget } from './types';++export class TargetBuffer {+ public target: Target;+ public fastStart: false | 'in-memory';+ public handle: FileSystemFileHandle | WritableFileTarget | undefined;++ public constructor(+ target: StreamTarget | BufferTarget,+ fastStart: false | 'in-memory',+ handle: FileSystemFileHandle | WritableFileTarget | undefined+ ) {+ this.target = target;+ this.fastStart = fastStart;+ this.handle = handle;+ }++ public static async create(+ handle?: FileSystemFileHandle | WritableFileTarget+ ): Promise<TargetBuffer> {+ if (handle) {+ const fileStream = await handle.createWritable();+ return new TargetBuffer(new StreamTarget(fileStream, { chunked: true }), false, handle);+ }++ return new TargetBuffer(new BufferTarget(), 'in-memory', undefined);+ }++ public async close(format: ContainerFormat = 'mp4'): Promise<Blob | undefined> {+ if (this.fastStart === 'in-memory' && this.target instanceof BufferTarget) {+ if (!this.target.buffer) return;+ return new Blob([this.target.buffer], { type: mimeTypes[format] });+ }++ return undefined;+ }+}++const mimeTypes: Record<ContainerFormat, string> = {+ mp4: 'video/mp4',+ webm: 'video/webm',+ ogg: 'audio/ogg',+ mov: 'video/quicktime',+};diff --git a/packages/encoder/src/encoder.ts b/packages/encoder/src/encoder.tsnew file mode 100644index 00000000..7ea3404d--- /dev/null+++ b/packages/encoder/src/encoder.ts@@ -0,0 +1,511 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import {+ CanvasSource,+ Output,+ AudioSample,+ AudioSampleSource,+} from 'mediabunny';+import { Not, Or } from 'koota';+import {+ createRuntimeWorld, switchActiveScene, serializeEntity, cloneFromRecords,+ getEntityTree, getDocument, propagateTimeRangeDown, framesToSeconds,+ assert, store, disposeDecoders,+ playbackSystem, motionSystem, transformSystem, renderSystem,+ AudioBus, AudioBusHandle,+ ChildOf, Deleted, Geometry, Group, Paint, Workarea, Playback,+ AudioPlayback, Computed, Delay, Trim, Transition, Keyframe, Animation,+ Position, Offset, Rotation, Scale, Skew,+ Project, Mode, Time, FrameRate, Camera, RenderSurface, AudioEngine,+ Assets, Fonts, FramePromises,+} from '@diffusionstudio/runtime';++import { TargetBuffer } from './buffer';+import { createOutputFormat } from './format';+import { createRenderEventDetail } from './utils';++import type { World } from 'koota';+import type { EncoderConfig } from './interfaces';+import type { ExportResult } from './types';++export async function createEncoder(sourceWorld: World, config: EncoderConfig) {+ const sourceComputed = store(sourceWorld, Computed);+ const sourceScene = config.scene;+ const sourceSceneId = sourceScene.id();+ const sourceFrameRate = sourceWorld.get(FrameRate)?.value ?? 30;+ const frameRate = config.video?.fps ?? sourceFrameRate;+ const numberOfChannels = config.audio?.numberOfChannels ?? 2;+ const sampleRate = config.audio?.sampleRate ?? 48000;+ const sceneWidth = sourceComputed.width[sourceSceneId]!;+ const sceneHeight = sourceComputed.height[sourceSceneId]!;+ const sceneEnd = sourceComputed.end[sourceSceneId]!;++ // Honor the scene's Workarea trait: render only the frames between+ // Workarea.start and Workarea.end instead of the full scene duration.+ const hasWorkarea = sourceScene.has(Workarea);+ const sourceWorkarea = store(sourceWorld, Workarea);+ const workareaStart = hasWorkarea+ ? Math.max(0, Math.min(sceneEnd, sourceWorkarea.start[sourceSceneId] ?? 0))+ : 0;+ const workareaEnd = hasWorkarea+ ? Math.max(workareaStart, Math.min(sceneEnd, sourceWorkarea.end[sourceSceneId] ?? sceneEnd))+ : sceneEnd;+ const workareaFrames = workareaEnd - workareaStart;++ const playheadStartSeconds = framesToSeconds(workareaStart, sourceFrameRate);+ const duration = framesToSeconds(workareaFrames, sourceFrameRate);+ const audioBitrate = config.audio?.bitrate ?? 128e3;+ const videoBitrate = config.video?.bitrate ?? 10e6;+ const audioCodec = config.audio?.codec ?? 'aac';+ const resolution = config.video?.resolution ?? 1080;+ const scale = Math.round(resolution * 1e6 / sceneHeight) / 1e6;+ const width = Math.round(sceneWidth * scale / 2) * 2;+ const height = Math.round(sceneHeight * scale / 2) * 2;+ const videoCodec = config.video?.codec ?? 'avc';+ const containerFormat = config.format ?? 'mp4';+ const audioEnabled = config.audio?.enabled ?? true;+ const videoEnabled = config.video?.enabled ?? true;+ const frameDuration = 1 / frameRate;++ if (config.format === 'ogg') {+ config.audio = { ...config.audio, enabled: true };+ config.video = { ...config.video, enabled: false };+ }++ const sceneSubtree = getEntityTree(sourceWorld, sourceScene);+ const subtreeRecords = sceneSubtree.map(entity => serializeEntity(entity));++ // Offscreen canvas+ const offscreenCanvas = new OffscreenCanvas(width, height);+ const offscreenCtx = offscreenCanvas.getContext('2d')!;++ // Offline audio context+ const offlineAudioCtx = new OfflineAudioContext(+ numberOfChannels,+ Math.ceil(duration * sampleRate),+ sampleRate,+ );++ // Build a fresh offline world that shares assets with the source. Fonts are+ // already loaded into the document; copying the array just tracks them as+ // loaded so the new world doesn't try to re-add them.+ const world = createRuntimeWorld(sourceWorld.get(Project)?.id ?? '');+ world.set(Mode, { value: videoEnabled ? 'offline-video' : 'offline-audio' });+ world.set(Assets, sourceWorld.get(Assets)!);+ world.set(Fonts, { list: [...(sourceWorld.get(Fonts)?.list ?? [])] });+ world.set(RenderSurface, { canvas: offscreenCanvas, ctx: offscreenCtx, resolution: scale });+ world.set(AudioEngine, { context: offlineAudioCtx });+ world.set(Camera, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 });+ world.set(FrameRate, { value: frameRate });+ world.set(FramePromises, { list: [] });++ const eidMap = cloneFromRecords(world, subtreeRecords);+ const scene = eidMap.get(sourceScene);+ assert(scene !== undefined, 'Failed to clone scene subtree');++ rescaleFrameTimestamps(world, sourceFrameRate, frameRate);+ recomputeAllTimeRanges(world);+ normalizeSceneTransform(world, scene.id());++ switchActiveScene(world, scene);++ const computed = store(world, Computed);+ const playback = store(world, Playback);+ // The workarea (Workarea trait) is runtime-only and never serialized, so it+ // isn't cloned into the offline world — nothing to strip here. The encoder+ // drives the playhead explicitly within the [workareaStart, workareaEnd]+ // window computed from the source scene above.+ const sceneId = scene.id();+ computed.localTimeInSeconds[sceneId] = playheadStartSeconds;+ computed.localTime[sceneId] = Math.round(playheadStartSeconds * frameRate);+ playback.playing[sceneId] = true;+ playback.loop[sceneId] = false;+ playback.speed[sceneId] = 1;+ // Anchor audio scheduling: context starts at t=0 but the playhead starts at+ // playheadStartSeconds, so audioDelay = -playheadStartSeconds shifts each+ // entity's scheduled audio back into the encoded window.+ if (!scene.has(AudioPlayback)) scene.add(AudioPlayback);+ const audioPlayback = store(world, AudioPlayback);+ audioPlayback.contextOffsetInSeconds[sceneId] = 0;+ audioPlayback.timelineOffsetInSeconds[sceneId] = playheadStartSeconds;++ // Re-execute any mounts into this offline world so it owns its own reactive+ // graphs + runtime hosts (surface/html). The per-frame playbackSystem then+ // drives them via the Mounts trait, so ticker-animated surfaces render.+ const mounts = videoEnabled ? (await config.realizeMounts?.(world)) ?? null : null;++ // Set up mediabunny output+ const buffer = await TargetBuffer.create(config.target);+ const format = await createOutputFormat(buffer, config.format);+ const output = new Output({ format, target: buffer.target });++ if (config.comment) {+ output.setMetadataTags({ comment: config.comment });+ }++ const audioSource = new AudioSampleSource({+ codec: audioCodec,+ bitrate: audioBitrate,+ });++ const sceneFills = [...world.query(Paint, ChildOf(scene), Not(Deleted))];++ const videoSource = new CanvasSource(offscreenCanvas, {+ codec: videoCodec,+ bitrate: videoBitrate,+ latencyMode: 'quality',+ alpha: containerFormat === 'webm' && !sceneFills.length+ ? 'keep'+ : 'discard',+ });++ if (audioEnabled) {+ output.addAudioTrack(audioSource);+ }++ if (videoEnabled) {+ output.addVideoTrack(videoSource);+ }++ // Audio worklet streams render-quantum chunks back to the main thread so+ // we can hand them to mediabunny in step with the visual frame loop.+ const audioWorkletUrl = createAudioWorkletUrl();+ await offlineAudioCtx.audioWorklet.addModule(audioWorkletUrl);++ const sharedBuffer = new SharedArrayBuffer(4);+ const sharedUint32Array = new Uint32Array(sharedBuffer);+ const mixNode = offlineAudioCtx.createGain();++ const sinkNode = new AudioWorkletNode(offlineAudioCtx, 'sink', {+ channelCount: numberOfChannels,+ channelCountMode: 'explicit',+ numberOfInputs: 1,+ numberOfOutputs: 1,+ outputChannelCount: [numberOfChannels],+ processorOptions: {+ buffer: sharedUint32Array,+ },+ });++ mixNode.connect(sinkNode);+ sinkNode.connect(offlineAudioCtx.destination);++ // connect the scene bus to the mix node+ const sceneBus = new AudioBus(world, scene);+ scene.add(AudioBusHandle);+ scene.set(AudioBusHandle, sceneBus);+ sceneBus.connect(mixNode);++ let sampleIndex = 0;+ let audioSourcePromise: Promise<void> | null = null;++ const allAudioReceived = Promise.withResolvers<void>();++ sinkNode.port.onmessage = event => {+ if (output?.state === 'canceled') {+ return;+ }++ const planarBuffer = event.data as Float32Array;++ if (planarBuffer.length === 0) {+ allAudioReceived.resolve();+ return;+ }++ const sampleCount = planarBuffer.length / numberOfChannels;++ const audioSample = new AudioSample({+ data: planarBuffer,+ format: 'f32-planar',+ numberOfChannels,+ sampleRate,+ timestamp: sampleIndex / sampleRate,+ });+ audioSourcePromise = audioSource.add(audioSample).finally(() => {+ audioSample.close();+ });++ sampleIndex += sampleCount;+ };++ const emitProgress = createThrottledCallback(config.onProgress, 1000 / 3); // 3 times per second++ let canceled = false;+ const cancel = () => (canceled = true);++ const render = async (): Promise<ExportResult> => {+ try {+ await output.start();+ const start = performance.now();+ const startTime = start;+ const totalFrames = Math.floor(duration * frameRate);++ let audioRenderingDone = false;+ let audioRenderingCompleted: Promise<AudioBuffer> | null = null;+ if (audioEnabled) {+ audioRenderingCompleted = offlineAudioCtx.startRendering();+ audioRenderingCompleted.then(() => { audioRenderingDone = true; });+ }++ let lastAudioSampleCount = 0;++ for (let frame = 0; frame < totalFrames; frame++) {+ if (canceled) {+ await output.cancel();+ Atomics.store(sharedUint32Array, 0, 2 ** 31 - 1);+ return { type: 'canceled' };+ }++ const timestamp = frame * frameDuration;+ const playheadSeconds = playheadStartSeconds + timestamp;++ // advancePlayhead is a no-op in offline mode; the encoder owns the+ // playhead and writes both seconds + frames before each tick.+ computed.localTimeInSeconds[sceneId] = playheadSeconds;+ computed.localTime[sceneId] = Math.round(playheadSeconds * frameRate);+ const time = world.get(Time)!;+ world.set(Time, { delta: frameDuration * 1000, now: time.now + frameDuration * 1000 });++ {+ playbackSystem(world);+ await resolverSystem(world);+ motionSystem(world);+ }++ if (videoEnabled) {+ // only visual systems+ transformSystem(world);+ renderSystem(world);+ }++ if (videoEnabled) {+ await videoSource.add(timestamp, frameDuration);+ }++ if (audioEnabled) {+ await audioSourcePromise;++ while (Atomics.load(sharedUint32Array, 0) > sampleRate && !audioRenderingDone) {+ await new Promise(resolve => setTimeout(resolve, 0));+ }++ const totalAudioSampleCount = Math.floor(timestamp * sampleRate);+ const diff = totalAudioSampleCount - lastAudioSampleCount;+ Atomics.add(sharedUint32Array, 0, diff);+ lastAudioSampleCount = totalAudioSampleCount;+ }++ emitProgress(createRenderEventDetail(frame, totalFrames, startTime));+ }++ console.info(+ `Encoded frames at ${((totalFrames * 1000) / (performance.now() - start)).toFixed(2)}FPS`+ );++ if (audioEnabled) {+ assert(audioRenderingCompleted !== null, 'Audio rendering was not started');++ Atomics.add(+ sharedUint32Array,+ 0,+ Math.ceil(duration * sampleRate) + sampleRate - lastAudioSampleCount,+ );++ await audioRenderingCompleted;++ sinkNode.port.postMessage(null);+ await allAudioReceived.promise;+ }++ console.info('Finalizing file');++ await output.finalize();++ console.info('Export complete');++ return {+ type: 'success',+ data: await buffer.close(containerFormat),+ };+ } catch (e) {+ return {+ type: 'error',+ error: e instanceof Error ? e : new Error('Unknown error'),+ };+ } finally {+ URL.revokeObjectURL(audioWorkletUrl);+ // Free the graphs + hosts realized above (this offline world is+ // discarded, so nothing else disposes them — an HtmlHost otherwise+ // leaks a canvas on document.body).+ mounts?.disposeAll();+ // Release decoder file handles/caches, then the world slot itself+ // (koota caps live worlds at 16, so throwaway offline worlds must+ // be destroyed — bitecs could just drop them).+ disposeDecoders(world, getDocument(world));+ world.destroy();+ }+ };++ return {+ render,+ cancel,+ };+}+++function audioWorkletCode() {+ // eslint-disable-next-line no-undef+ class SinkProcessor extends AudioWorkletProcessor {+ buffer: Uint32Array;++ constructor(options: {+ processorOptions: {+ buffer: Uint32Array;+ };+ }) {+ super();++ this.buffer = options.processorOptions.buffer;++ this.port.onmessage = () => {+ this.port.postMessage(new Float32Array(0));+ };+ }++ process(inputs: Float32Array[][], outputs: Float32Array[][]) {+ const input = inputs[0];+ const output = outputs[0];+ const renderQuantum = output[0].length;++ while (Atomics.load(this.buffer, 0) < renderQuantum);++ const planarBuffer = new Float32Array(output.length * output[0].length);+ for (let channel = 0; channel < Math.min(input.length, output.length); channel++) {+ planarBuffer.set(input[channel], channel * output[0].length);+ }++ this.port.postMessage(planarBuffer, [planarBuffer.buffer]);++ Atomics.sub(this.buffer, 0, renderQuantum);++ return true;+ }+ }++ // eslint-disable-next-line no-undef+ registerProcessor('sink', SinkProcessor);+}++function createAudioWorkletUrl(): string {+ const blob = new Blob([`(${audioWorkletCode.toString()})()`], { type: 'application/javascript' });+ return URL.createObjectURL(blob);+}++function createThrottledCallback<T>(callback: ((value: T) => void) | undefined, intervalMs: number): (value: T) => void {+ let lastCalledAt = -Infinity;++ return (value: T) => {+ const now = performance.now();+ if (now - lastCalledAt < intervalMs) {+ return;+ }++ lastCalledAt = now;+ callback?.(value);+ };+}++function rescaleFrameTimestamps(+ world: World,+ sourceFrameRate: number,+ targetFrameRate: number,+): void {+ if (sourceFrameRate === targetFrameRate) return;++ const ratio = targetFrameRate / sourceFrameRate;+ const rescale = (v: number) => Math.round(v * ratio);++ const delay = store(world, Delay);+ const trim = store(world, Trim);+ const transition = store(world, Transition);+ const keyframe = store(world, Keyframe);+ const animation = store(world, Animation);++ for (const entity of world.query(Or(Delay, Trim, Transition, Keyframe, Animation))) {+ const eid = entity.id();+ if (entity.has(Delay)) {+ delay.value[eid] = rescale(delay.value[eid] ?? 0);+ }+ if (entity.has(Trim)) {+ trim.start[eid] = rescale(trim.start[eid] ?? 0);+ trim.end[eid] = rescale(trim.end[eid] ?? 0);+ }+ if (entity.has(Transition)) {+ transition.duration[eid] = rescale(transition.duration[eid] ?? 0);+ }+ if (entity.has(Keyframe)) {+ keyframe.time[eid] = rescale(keyframe.time[eid] ?? 0);+ }+ if (entity.has(Animation)) {+ animation.duration[eid] = rescale(animation.duration[eid] ?? 0);+ animation.delay[eid] = rescale(animation.delay[eid] ?? 0);+ }+ }+}++function normalizeSceneTransform(world: World, sceneId: number): void {+ const position = store(world, Position);+ const offset = store(world, Offset);+ const rotation = store(world, Rotation);+ const scaleStore = store(world, Scale);+ const skew = store(world, Skew);+ position.x[sceneId] = 0;+ position.y[sceneId] = 0;+ offset.x[sceneId] = 0;+ offset.y[sceneId] = 0;+ rotation.value[sceneId] = 0;+ scaleStore.x[sceneId] = 1;+ scaleStore.y[sceneId] = 1;+ skew.x[sceneId] = 0;+ skew.y[sceneId] = 0;+ // computeLocalMatrix reads the Computed.* mirror, not the raw traits. The+ // base traits are populated via entity.set during the clone, which fires+ // the observer that copies the (canvas) position into Computed. Writing+ // the raw stores above bypasses that observer, so the Computed mirror must+ // be reset here too — otherwise the scene renders at its canvas position+ // instead of centered in the export.+ const computed = store(world, Computed);+ computed.positionX[sceneId] = 0;+ computed.positionY[sceneId] = 0;+ computed.offsetX[sceneId] = 0;+ computed.offsetY[sceneId] = 0;+ computed.rotation[sceneId] = 0;+ computed.scaleX[sceneId] = 1;+ computed.scaleY[sceneId] = 1;+ computed.skewX[sceneId] = 0;+ computed.skewY[sceneId] = 0;+}++export async function resolverSystem(world: World) {+ const promises = world.get(FramePromises)?.list;+ if (promises?.length) {+ await Promise.all(promises.filter(promise => promise !== null));+ world.set(FramePromises, { list: [] });+ }+}++/**+ * Refresh every entity's accumulated delay + time bounds after raw store+ * rewrites. propagateTimeRangeDown recomputes top-down and then re-derives+ * each parent after its children (bitecs used a two-pass Hierarchy(ChildOf)+ * walk; koota has no hierarchy-ordered queries).+ */+export function recomputeAllTimeRanges(world: World): void {+ const document = getDocument(world);+ for (const node of world.query(Or(Geometry, Group), ChildOf(document), Not(Deleted))) {+ propagateTimeRangeDown(world, node);+ }+}diff --git a/packages/encoder/src/format.ts b/packages/encoder/src/format.tsnew file mode 100644index 00000000..1b831363--- /dev/null+++ b/packages/encoder/src/format.ts@@ -0,0 +1,25 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { TargetBuffer } from './buffer';+import type { ContainerFormat } from './types';++/**+ * Lazily imports the output format based on the format specified+ */+export async function createOutputFormat(buffer: TargetBuffer, format?: ContainerFormat) {+ if (format == 'webm') {+ const { WebMOutputFormat } = await import('mediabunny');+ return new WebMOutputFormat();+ } else if (format == 'ogg') {+ const { OggOutputFormat } = await import('mediabunny');+ return new OggOutputFormat();+ } else if (format == 'mov') {+ const { MovOutputFormat } = await import('mediabunny');+ return new MovOutputFormat();+ } else {+ const { Mp4OutputFormat } = await import('mediabunny');+ return new Mp4OutputFormat({ fastStart: buffer.fastStart });+ }+}diff --git a/packages/encoder/src/image-encoder.ts b/packages/encoder/src/image-encoder.tsnew file mode 100644index 00000000..0f89df65--- /dev/null+++ b/packages/encoder/src/image-encoder.ts@@ -0,0 +1,241 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Not, Or } from 'koota';+import {+ createRuntimeWorld, createEntity, appendChild, serializeEntity,+ cloneFromRecords, getEntityTree, getDocument, framesToSeconds,+ formatTimecode, assert, store, disposeDecoders,+ playbackSystem, motionSystem, transformSystem, renderSystem,+ ChildOf, Deleted, Geometry, Group, Hidden, IsMask, Muted, Culled,+ ClipsContent, Playback, Computed, WorldBounds,+ Project, Mode, Camera, RenderSurface, AudioEngine, Assets, Fonts,+ FramePromises, Time, FrameRate,+} from '@diffusionstudio/runtime';++import { resolverSystem, recomputeAllTimeRanges } from './encoder';++import type { Entity, World } from 'koota';+import type { AABB } from '@diffusionstudio/runtime';+import type { ImageEncoderConfig } from './interfaces';++/** One capture: the PNG plus the timecode of the frame rendered, e.g. `01s15f`. */+export type CapturedImage = { base64: string; timecode: string };++export type ImageExportResult =+ | { type: 'success'; data: CapturedImage[] }+ | { type: 'canceled' }+ | { type: 'error'; error: Error };++/**+ * Renders single frames of one entity into standalone PNGs (base64, no+ * data-url prefix). The entity's subtree is cloned into a fresh offline+ * world and drawn at the top level: the canvas is sized to the union of the+ * entity's world-space AABBs across the requested frames and the camera+ * shifts that box onto the canvas, so the node is fully visible regardless+ * of where it sat in its source scene.+ */+export async function createImageEncoder(sourceWorld: World, config: ImageEncoderConfig) {+ assert(config.frames.length > 0, 'No frames requested');++ const subtree = getEntityTree(sourceWorld, config.entity);+ const records = subtree.map(entity => serializeEntity(entity));++ // Re-root the node. Its parent isn't part of the clone, so a dangling+ // ChildOf would attach to an arbitrary entity in the new world; Delay and+ // Hidden only mean something on the timeline it was lifted out of.+ const rootRecord = records.find(record => record.eid === config.entity);+ assert(rootRecord !== undefined, `No such entity: ${config.entity}`);+ delete rootRecord.ChildOf;+ delete rootRecord.Delay;+ delete rootRecord.Hidden;++ const offscreenCanvas = new OffscreenCanvas(2, 2);+ const offscreenCtx = offscreenCanvas.getContext('2d')!;++ // Never started — image capture is video-only; the context just satisfies+ // world construction and lazy audio-bus wiring.+ const offlineAudioCtx = new OfflineAudioContext(2, 1, 48000);++ const sourceFrameRate = sourceWorld.get(FrameRate)?.value ?? 30;+ const world = createRuntimeWorld(sourceWorld.get(Project)?.id ?? '');+ world.set(Mode, { value: 'offline-video' });+ world.set(Assets, sourceWorld.get(Assets)!);+ world.set(Fonts, { list: [...(sourceWorld.get(Fonts)?.list ?? [])] });+ world.set(RenderSurface, { canvas: offscreenCanvas, ctx: offscreenCtx, resolution: 1 });+ world.set(AudioEngine, { context: offlineAudioCtx });+ world.set(Camera, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 });+ world.set(FrameRate, { value: sourceFrameRate });+ world.set(FramePromises, { list: [] });++ const eidMap = cloneFromRecords(world, records);+ const root = eidMap.get(config.entity);+ assert(root !== undefined, 'Failed to clone entity subtree');++ const mounts = (await config.realizeMounts?.(world)) ?? null;++ const cloned = [...eidMap.values()];+ const computed = store(world, Computed);+ const playback = store(world, Playback);+ const worldBounds = store(world, WorldBounds);++ // Mute everything so the playback system never initializes audio decoders.+ for (const entity of cloned) {+ entity.add(Muted);+ }++ // The motion system only samples parented entities, so the node's own+ // keyframes would be skipped as a top-level root. A synthetic group above+ // it owns the playhead instead and keeps the node an ordinary child.+ const stage = createEntity(world);+ stage.add(Group);+ appendChild(world, root, stage);++ recomputeAllTimeRanges(world);++ // Map "frame 0 = first visible frame": with Delay stripped the node's+ // computed start is 0 unless it carries a Trim, whose start offsets it.+ const startFrame = computed.start[root.id()] ?? 0;++ // Entities in the clone that own a timeline clock+ const clocks = [...world.query(Playback, Not(Deleted))];++ const seek = (frame: number) => {+ const stageFrame = startFrame + frame;+ const stageSeconds = framesToSeconds(stageFrame, sourceFrameRate);+ computed.localTime[stage.id()] = stageFrame;+ computed.localTimeInSeconds[stage.id()] = stageSeconds;++ for (const clock of clocks) {+ computed.localTime[clock.id()] = stageFrame;+ computed.localTimeInSeconds[clock.id()] = stageSeconds;+ playback.playing[clock.id()] = true;+ }++ const delta = 1000 / sourceFrameRate;+ const time = world.get(Time)!;+ world.set(Time, { delta, now: time.now + delta });++ // Cull flags derived from a stale canvas/camera would make the motion+ // system skip entities and keep decoders idle — clear them every tick.+ for (const entity of cloned) {+ if (entity.has(Culled)) entity.remove(Culled);+ }++ playbackSystem(world);+ };++ // Union an entity's world-space AABB into `bounds`, descending so children+ // that overflow a leaf's own rect still count — except where content is+ // clipped. Group bounds already aggregate their children.+ const measure = (entity: Entity, bounds: AABB): void => {+ if (entity.has(Hidden) || entity.has(IsMask)) return;+ if (computed.visibility[entity.id()] === 0) return;++ bounds.minX = Math.min(bounds.minX, worldBounds.minX[entity.id()]!);+ bounds.minY = Math.min(bounds.minY, worldBounds.minY[entity.id()]!);+ bounds.maxX = Math.max(bounds.maxX, worldBounds.maxX[entity.id()]!);+ bounds.maxY = Math.max(bounds.maxY, worldBounds.maxY[entity.id()]!);++ if (entity.has(ClipsContent)) return;+ for (const child of world.query(Or(Geometry, Group), ChildOf(entity), Not(Deleted))) {+ measure(child, bounds);+ }+ };++ // Warm-up tick: group bounds aggregate from their children's local+ // matrices, which don't exist until the first transform walk has visited+ // them — measuring on that walk would union stale boxes.+ seek(config.frames[0]!);+ motionSystem(world);+ transformSystem(world);++ // Measure pass: run motion + transforms over every requested frame with an+ // identity camera and union the bounds, so an animated node stays fully+ // visible in a fixed-size output across all frames.+ const bounds: AABB = { minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity };+ for (const frame of config.frames) {+ seek(frame);+ motionSystem(world);+ transformSystem(world);+ measure(root, bounds);+ }+ assert(Number.isFinite(bounds.minX), 'Entity is not visible at any of the requested frames');++ const boundsWidth = Math.max(1, bounds.maxX - bounds.minX);+ const boundsHeight = Math.max(1, bounds.maxY - bounds.minY);++ /** Re-target the output height; callers that lay frames out do this once measured. */+ const resize = (height?: number) => {+ const scale = height ? height / boundsHeight : 1;+ offscreenCanvas.width = Math.max(1, Math.round(boundsWidth * scale));+ offscreenCanvas.height = Math.max(1, Math.round(boundsHeight * scale));+ world.set(RenderSurface, { resolution: scale });+ };++ resize(config.resolution);+ // Shift the measured box onto the canvas so the node is centered in view;+ // the camera translation is scaled by the surface resolution at render time.+ world.set(Camera, { e: -bounds.minX, f: -bounds.minY });++ let canceled = false;+ const cancel = () => (canceled = true);++ const render = async (): Promise<ImageExportResult> => {+ try {+ const images: CapturedImage[] = [];++ for (const frame of config.frames) {+ if (canceled) {+ return { type: 'canceled' };+ }++ seek(frame);+ await resolverSystem(world);+ motionSystem(world);+ transformSystem(world);+ renderSystem(world);++ const stampFrame = startFrame + frame;+ images.push({+ base64: await toBase64Png(offscreenCanvas),+ timecode: formatTimecode(framesToSeconds(stampFrame, sourceFrameRate), sourceFrameRate),+ });+ }++ return { type: 'success', data: images };+ } catch (e) {+ return {+ type: 'error',+ error: e instanceof Error ? e : new Error('Unknown error'),+ };+ } finally {+ // Free the graphs + hosts realized above (this offline world is+ // discarded; realized HtmlHosts otherwise leak a canvas on+ // document.body), then release decoders and the world slot itself+ // (koota caps live worlds at 16).+ mounts?.disposeAll();+ disposeDecoders(world, getDocument(world));+ world.destroy();+ }+ };++ return {+ render,+ cancel,+ resize,+ bounds: { width: boundsWidth, height: boundsHeight },+ };+}++async function toBase64Png(canvas: OffscreenCanvas): Promise<string> {+ const blob = await canvas.convertToBlob({ type: 'image/png' });+ const dataUrl = await new Promise<string>((resolve, reject) => {+ const reader = new FileReader();+ reader.onload = () => resolve(reader.result as string);+ reader.onerror = () => reject(reader.error);+ reader.readAsDataURL(blob);+ });+ return dataUrl.split(',')[1] ?? '';+}diff --git a/packages/encoder/src/index.ts b/packages/encoder/src/index.tsnew file mode 100644index 00000000..87d54003--- /dev/null+++ b/packages/encoder/src/index.ts@@ -0,0 +1,11 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++export * from './buffer';+export * from './encoder';+export * from './format';+export * from './image-encoder';+export * from './interfaces';+export * from './types';+export * from './utils';diff --git a/packages/encoder/src/interfaces.ts b/packages/encoder/src/interfaces.tsnew file mode 100644index 00000000..57a539e6--- /dev/null+++ b/packages/encoder/src/interfaces.ts@@ -0,0 +1,155 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { VideoCodec, AudioCodec } from 'mediabunny';+import type { Entity, World } from 'koota';+import type { ContainerFormat, EncoderProgress, RealizedMounts, WritableFileTarget } from './types';++export interface AudioConfig {+ /**+ * Enable audio encoding+ * @default true+ */+ enabled?: boolean;++ /**+ * A floating point number indicating the audio context's sample rate, in samples per second.+ * @default 48000+ */+ sampleRate?: number;++ /**+ * Defines the number of channels of the composed audio+ * @default 2+ */+ numberOfChannels?: number;++ /**+ * Defines the bitrate at which the audio should be rendered at+ * @default 128e3+ */+ bitrate?: number;++ /**+ * Defines the codec to use for the audio+ * @default 'aac'+ */+ codec?: AudioCodec;+}++export interface VideoConfig {+ /**+ * Defines the codec to use for the video+ * @default 'avc'+ */+ codec?: VideoCodec;++ /**+ * The full codec string as specified in the WebCodecs API Codec Registry.+ */+ fullCodecString?: string;++ /**+ * Enable video encoding+ * @default true+ */+ enabled?: boolean;++ /**+ * Defines the bitrate at which the video should be rendered at+ * @default 10e6+ */+ bitrate?: number;++ /**+ * Defines the fps at which the composition will be rendered+ * @default 30+ */+ fps?: number;++ /**+ * Resolution of the composition+ * @example+ * 1080 for 1080p+ * 1440 for 1440p+ * 2160 for 4k+ * @default 1080+ */+ resolution?: number;+}++export interface EncoderConfig {+ /**+ * Video encoding configuration+ */+ video?: VideoConfig;++ /**+ * Audio encoding configuration+ */+ audio?: AudioConfig;++ /**+ * Defines the output format of the encoded file+ * @default 'mp4'+ */+ format?: ContainerFormat;++ /**+ * Defines the range of the composition to encode+ */+ trim?: {+ /**+ * Defines the time when the encoding should end+ * @default scene.duration+ */+ end?: number;+ };++ /**+ * Defines the target file to encode to.+ * When a FileSystemFileHandle (or any WritableFileTarget) is provided, the+ * output is streamed to the file. When omitted, the output is buffered in+ * memory and returned as a Blob in the ExportResult.+ */+ target?: FileSystemFileHandle | WritableFileTarget;++ /**+ * Defines the progress callback+ */+ onProgress?: (progress: EncoderProgress) => void;++ /**+ * Defines the source scene to encode+ */+ scene: Entity;++ /**+ * Re-executes the source world's mounts into the offline world so their+ * reactive graphs and runtime hosts (surface/html) render during capture.+ * Mount realization needs the host app's module loader, so it is injected.+ */+ realizeMounts?: (world: World) => Promise<RealizedMounts | null>;++ /**+ * Metadata comment tag written into the container (e.g. an app version).+ */+ comment?: string;+}++export interface ImageEncoderConfig {+ /**+ * The source entity to capture+ */+ entity: Entity;++ /** Frames to capture, relative to the entity's first visible frame. */+ frames: number[];++ /** Target output height in px (default: the entity's native size). */+ resolution?: number;++ /** See EncoderConfig.realizeMounts. */+ realizeMounts?: (world: World) => Promise<RealizedMounts | null>;+}diff --git a/packages/encoder/src/types.ts b/packages/encoder/src/types.tsnew file mode 100644index 00000000..e584cd8b--- /dev/null+++ b/packages/encoder/src/types.ts@@ -0,0 +1,36 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { StreamTargetChunk } from 'mediabunny';++// The container format is part of the persisted document model (the+// ExportSettings trait), so the runtime owns the type.+export type { ContainerFormat } from '@diffusionstudio/runtime';++// Minimal FileSystemFileHandle-shaped target+export interface WritableFileTarget {+ createWritable(): Promise<WritableStream<StreamTargetChunk>>;+}++export type EncoderProgress = {+ total: number;+ progress: number;+ remaining: Date;+};++export type ExportResult =+ | {+ type: 'success';+ data: Blob | undefined;+ }+ | {+ type: 'canceled';+ }+ | {+ type: 'error';+ error: Error;+ };++/** Mount graphs re-executed into an offline world (see EncoderConfig.realizeMounts). */+export type RealizedMounts = { disposeAll(): void };diff --git a/packages/encoder/src/utils.ts b/packages/encoder/src/utils.tsnew file mode 100644index 00000000..99496a5a--- /dev/null+++ b/packages/encoder/src/utils.ts@@ -0,0 +1,22 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Helper for creating the render event detail+ */+export function createRenderEventDetail(progress: number, total: number, startTime: number) {+ const duration = performance.now() - startTime;+ const time = (duration / gte1(progress)) * (total - progress);+ const remaining = new Date(time);++ return { remaining, progress, total };+}++/**+ * Helper for making sure a number is greater than 1+ */+function gte1(num: number): number {+ if (num < 1) return 1;+ return num;+}diff --git a/packages/encoder/tsconfig.json b/packages/encoder/tsconfig.jsonnew file mode 100644index 00000000..c150db11--- /dev/null+++ b/packages/encoder/tsconfig.json@@ -0,0 +1,10 @@+{+ "extends": "../../tsconfig.base.json",+ "compilerOptions": {+ "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo",+ "erasableSyntaxOnly": false,+ "lib": ["es2024", "DOM", "DOM.Iterable"],+ "types": ["@webgpu/types", "@types/audioworklet"]+ },+ "include": ["src"]+}diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.mdindex c0001200..f38d7721 100644--- a/packages/runtime/MIGRATION.md+++ b/packages/runtime/MIGRATION.md@@ -38,6 +38,10 @@ Working doc for migrating the editor engine from bitecs (in `apps/web/src/compon| Fonts | `src/fonts/{fixtures,utils}.ts` | WebFonts fixtures verbatim; `loadWebFont` uses the Fonts world trait, resolves the FontFaceSet via globalThis (worker-capable), guards on missing FontFace (headless), and drops db persistence (app wraps it). `getLocalFonts` + `restoreFonts` stay app-side || Support utils | `src/utils/{color,async}.ts`, `clamp` in math | parseColor/colorToHex/rgbToColor (colord dep), AsyncMutex/attempt || Playback system | `src/systems/playback.ts`, `src/utils/transition.ts` | Playhead advance, visibility/local-time recursion, decoder forwarding, transitions, solo muting, live mounts step; `resolveAudioBus` exported from here. See semantic notes |+| Render system | `src/systems/render.ts`, `HitRegions` world trait | Full traversal port: paints (solid/gradient/media/waveform/shader/html/surface), shadows/strokes/effects, masks, ClipsContent, transitions, Generating pulse. No-op without a RenderSurface. See semantic notes |+| System schedule | `src/systems/schedule.ts` | `runSystems(world)`: playback → motion → transform → render, shared by the realtime tick and the capture loop |+| Encoders | `packages/encoder` (`@diffusionstudio/encoder`) | `engine/encode/*` extracted into its OWN package, not `src/capture/`: createEncoder + createImageEncoder as pure consumers of the runtime's public API. See semantic notes |+| Timecode helpers | `src/utils/time.ts` | `formatTimecode`/`formatTimestamp` (from `utils/transcode.ts`; the image encoder stamps captures with them) || Seeds (types only, rest follows) | `src/utils/text.ts` (Token/Line/TokenOptions), `src/capture/format.ts` (ContainerFormat) | The remaining engine files merge into these on their slice |Verification style used so far: `npx tsc --noEmit` in the package, plus throwaway `smoke-test.tmp.ts` in the package root run with `npm exec --yes tsx@latest -- smoke-test.tmp.ts`, deleted afterwards (no test infra in the package yet). koota caps live worlds at 16; call `world.destroy()` in test loops.@@ -74,7 +78,11 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All- DOM caveats, all lazy (nothing touches DOM at import time): HtmlHost/SurfaceHost create elements at construction (html-in-canvas is Chromium-flag-only anyway), ShaderHost needs WebGPU, `loadWebFont` no-ops without FontFace. tsconfig gained `DOM.Iterable` + `@webgpu/types`.- `SequenceDecoder` iterates the structural `AssetDirectoryHandle` (entries with `getFile`) instead of concrete FileSystemFileHandle, so node hosts can feed frame directories.- Playback: the bitecs top-level query `Not(ChildOf('*'))` became `ChildOf(document)`. Playhead advance, auto-stop, and visibility/localTime writes stay raw store writes (no change events, same as bitecs), and `AudioPlayback` is lazily added to playback roots on first tick. `resolveAudioBus` returns null when the world has no AudioEngine context (headless-without-audio; `AudioDecoder.playTo(null, ...)` already no-ops), and the context/offset reads guard the same way. The transition-cleanup `removeComponent(..., false)` became a plain `entity.remove(Transition)`; the app's history layer must treat Transition onRemove during a tick as system-driven, not undoable.-- `systems/schedule.ts` is still a stub: the realtime tick order is playback → motion → transform → render (per `api/engine.ts`), and the helper composing it lands with the render/capture slices when all members exist.+- Render: `world.hitRegions` became the `HitRegions` world trait. The render system pushes CALLBACK-LESS entries ({ kind: 'entity', id } for Interactive entities, { kind: 'hud', id: 'canvas', quad } for the stage in realtime); the bitecs handleGeometryInteraction/handleCanvasInteraction bindings are the app's: its input system clears the list each frame, pushes HUD controls with handlers attached, and maps callback-less targets to its default handlers. In headless worlds nothing pushes (Interactive is app-added, the canvas region is realtime-only), so the list cannot grow unbounded without an input system draining it.+- Render's top-level query `Not(ChildOf('*'))` became `ChildOf(document)`, like playback's. `getScaledImageProps` reads the ScaleMode trait against the ScaleModeType enum. `drawRectPath`, `renderFills`, and `renderNode` stay exported (HUD and thumbnail code consume them app-side).+- Encoders live in `@diffusionstudio/encoder` (deps: runtime, mediabunny, koota), which imports ONLY the runtime's public API — the boundary that motivated the extraction. `ContainerFormat` stays in the runtime (`src/capture/format.ts`) because the ExportSettings trait persists it; the encoder re-exports the type. Nothing inside the runtime may import the encoder (cycle), so encoder-driven utilities like `utils/captions.ts` stay app-side or move into the encoder package.+- Encoder API changes vs `engine/encode`: `config.scene`/`config.entity` are Entities; `realizeMounts` is injected via config (mount realization needs the host app's module loader + @diffusionstudio/jsx); the hardcoded "Made with Diffusion v..." metadata tag became `config.comment`; the bitecs `Hierarchy(ChildOf)` two-pass rescale walks became `recomputeAllTimeRanges` (exported) over `propagateTimeRangeDown` from the document's top-level nodes; and both encoders now dispose decoders and `world.destroy()` their throwaway offline worlds in `render()`'s finally (koota caps live worlds at 16 — bitecs could just drop them). Offline-world wiring is world traits (`Mode`/`Assets` shared by reference/`Fonts` copied/`RenderSurface`/`AudioEngine`/`FrameRate`/`FramePromises`) instead of the context bag; playhead/Playback/AudioPlayback writes stay raw store writes, matching the systems.+- The encoder package's tsconfig must mirror the runtime's (`DOM.Iterable`, `@webgpu/types`, plus `@types/audioworklet`): runtime exports point at TS sources, so consumers compile them with their own compiler settings.- `switchActiveScene` only sets the ActiveScene world trait; timeline buffer clearing and index rebuild stay app-side. `frameSelection` calls it once (bitecs called it redundantly inside and outside the transaction).- `insertAssetInTimeline` returns the created entity or null when no scene is active (no toast). Folder ops are sync and return the changed records (`deleteFolder` returns removed folder/asset ids) so the app can persist; `moveAssetsToFolder` updates records in place instead of re-fronting them like `saveAsset` did. Panel navigation state (current folder) stays app-side.- Decoder/host handle traits (`src/traits/media.ts`) are AoS traits typed as opaque `object` until the decoders move in; `entity.get(VideoDecoder)` returns the instance directly.@@ -93,11 +101,9 @@ Every world spawns one entity tagged `DocumentRoot` (`createRuntimeWorld`). All## Remaining work (suggested order)-1. **Capture** (`src/capture/`): `engine/encode/*` (encoder, image-encoder, buffer, format, utils). Uses Mode, FramePromises, RenderSurface. Fill `systems/schedule.ts` once render exists.-2. **Render system**: `systems/render.ts` is runtime-capable (OffscreenCanvas) and needed for headless capture; text rendering and gradients are already in (`utils/text.ts`, `systems/gradients.ts`; `addStopsTo` stayed private to gradients).-3. **Remaining utils**: captions, contact-sheet, audio-sync, transcode (audit each for DOM/app deps; DOM-touching ones stay app-side).-4. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.-5. **CLI**: point node capture at the runtime package.+1. **Remaining utils**: captions (needs the encoder package, so it lands there or app-side), contact-sheet, audio-sync, transcode (timecode helpers already moved; audit the rest for DOM/app deps; DOM-touching ones stay app-side).+2. **App rewrite** (`apps/web`): engine shell wires canvas/AudioContext/DPR into world traits, app-side world traits (input, HUD, selection tool, history, store, timelineIndex signal), history/persistence via koota events (including wrapping the folder/asset model ops with Dexie writes, font persistence around `loadWebFont` + `restoreFonts`/`getLocalFonts`, re-implementing `syncInteractiveState` over `ChildOf(document)` queries: `Not(ChildOf('*'))` throws in koota, and re-adding the editing-surface `syncKeyframeTrack` calls the observers slice dropped), Deleted-tombstone pruning + `unpersistEntity`, UI to `packages/koota-solid` hooks, timeline UI consumes new `TimelineNode.entity`. Delete `engine/components`, `engine/api` progressively.+3. **CLI**: point node capture at the runtime + encoder packages.## Layout remindersdiff --git a/packages/runtime/src/capture/format.ts b/packages/runtime/src/capture/format.tsindex 41110876..9127dd0c 100644--- a/packages/runtime/src/capture/format.ts+++ b/packages/runtime/src/capture/format.ts@@ -2,7 +2,8 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Container format for export. The rest of engine/encode/format.ts joins this-// file when capture moves into the runtime.+// Container format for export. Lives in the runtime (not the encoder+// package) because the ExportSettings trait persists it; the encoder+// imports it from here.export type ContainerFormat = 'mp4' | 'webm' | 'ogg' | 'mov';diff --git a/packages/runtime/src/capture/index.ts b/packages/runtime/src/capture/index.tsindex 68f561e5..af486a5b 100644--- a/packages/runtime/src/capture/index.ts+++ b/packages/runtime/src/capture/index.ts@@ -1,2 +1,3 @@-// Capture and encoding: encoder, buffer, image-encoder (move wholesale from engine/encode/).+// Persisted capture model. The encoders themselves live in+// @diffusionstudio/encoder; only what the document format needs stays here.export * from './format';diff --git a/packages/runtime/src/systems/index.ts b/packages/runtime/src/systems/index.tsindex ac687b3e..15f0ebb2 100644--- a/packages/runtime/src/systems/index.ts+++ b/packages/runtime/src/systems/index.ts@@ -3,3 +3,5 @@ export * from './transform';export * from './motion';export * from './gradients';export * from './playback';+export * from './render';+export * from './schedule';diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsnew file mode 100644index 00000000..aca6963e--- /dev/null+++ b/packages/runtime/src/systems/render.ts@@ -0,0 +1,868 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Render system (was systems/render.ts): draws the document tree onto the+// world's RenderSurface. Runs identically against the editor canvas and an+// OffscreenCanvas during capture; without a surface it is a no-op. Hit+// regions are pushed callback-less (see HitRegions); the app's input layer+// attaches its handlers.++import { Not, Or } from 'koota';+import { cubicBezier } from 'animejs';++import { store } from '../world/store';+import {+ COMPOSITE_OPERATIONS, EffectType, GeometryType, PaintType, ScaleModeType,+ StrokeCap, StrokeJoin, TransitionType,+} from '../constants';+import {+ ChildOf, Deleted, Hidden, Culled, Generating, Interactive, IsMask,+ ClipsContent, Geometry, Group, Paint, Caption, ScaleMode, Shader,+ Appearance, Effect, StrokeStyle, AssetId, Transition, MixedCornerRadius,+ LocalTransform, WorldTransform, Computed, Cache,+ HtmlHostHandle, SurfaceHostHandle,+ Mode, Time, FrameRate, Camera, Background, RenderSurface, Assets,+ HitRegions,+} from '../traits';+import { getDocument, getParentNode } from '../queries/hierarchy';+import { colorToHex } from '../utils/color';+import { renderText } from '../utils/text';+import { getTransitionWindow } from '../utils/transition';+import { createLinearGradient, createRadialGradient } from './gradients';+import {+ resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder,+ resolveCaptionDecoder, resolveShaderHost, getAudioPeaks,+} from '../media';++import type { Entity, World } from 'koota';+import type { Quad } from '../math/aabb';++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;++const MISSING_ASSET_COLOR = '#5C2828';++function getCtx(world: World): Ctx2D {+ return world.get(RenderSurface)!.ctx!;+}++export function drawRectPath(world: World, entity: Entity): void {+ const ctx = getCtx(world);+ const computed = store(world, Computed);+ const eid = entity.id();+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;++ const hasMixed = entity.has(MixedCornerRadius);+ let tl = hasMixed ? computed.cornerRadiusTopLeft[eid]! : computed.cornerRadius[eid]!;+ let tr = hasMixed ? computed.cornerRadiusTopRight[eid]! : tl;+ let br = hasMixed ? computed.cornerRadiusBottomRight[eid]! : tl;+ let bl = hasMixed ? computed.cornerRadiusBottomLeft[eid]! : tl;++ ctx.beginPath();++ if (tl === 0 && tr === 0 && br === 0 && bl === 0) {+ ctx.rect(0, 0, w, h);+ } else if (tl === tr && tr === br && br === bl) {+ ctx.roundRect(0, 0, w, h, tl);+ } else {+ // Clamp radii so adjacent corners don't exceed the edge length (CSS spec algorithm)+ const scale = Math.min(+ w / (tl + tr || 1),+ h / (tr + br || 1),+ w / (br + bl || 1),+ h / (bl + tl || 1),+ 1,+ );+ if (scale < 1) {+ tl *= scale;+ tr *= scale;+ br *= scale;+ bl *= scale;+ }++ ctx.moveTo(tl, 0);+ ctx.lineTo(w - tr, 0);+ if (tr > 0) ctx.arcTo(w, 0, w, tr, tr);+ else ctx.lineTo(w, 0);+ ctx.lineTo(w, h - br);+ if (br > 0) ctx.arcTo(w, h, w - br, h, br);+ else ctx.lineTo(w, h);+ ctx.lineTo(bl, h);+ if (bl > 0) ctx.arcTo(0, h, 0, h - bl, bl);+ else ctx.lineTo(0, h);+ ctx.lineTo(0, tl);+ if (tl > 0) ctx.arcTo(0, 0, tl, 0, tl);+ else ctx.lineTo(0, 0);+ }++ ctx.closePath();+}++function getScaledImageProps(+ mode: number,+ imgW: number,+ imgH: number,+ targetW: number,+ targetH: number,+): [dx: number, dy: number, sw: number, sh: number] {+ if (mode === ScaleModeType.FILL) {+ return [0, 0, targetW, targetH];+ }+ if (mode === ScaleModeType.FIT) {+ const scale = Math.min(targetW / imgW, targetH / imgH);+ const sw = imgW * scale;+ const sh = imgH * scale;+ return [(targetW - sw) / 2, (targetH - sh) / 2, sw, sh];+ }+ if (mode === ScaleModeType.COVER) {+ const scale = Math.max(targetW / imgW, targetH / imgH);+ const sw = imgW * scale;+ const sh = imgH * scale;+ return [(targetW - sw) / 2, (targetH - sh) / 2, sw, sh];+ }+ // ScaleModeType.NONE — original size+ return [0, 0, imgW, imgH];+}++const EPSILON = 1e-4;++/** Build a single CSS filter fragment from an effect sub-entity. Returns null if hidden or no-op. */+function effectFilter(world: World, sub: Entity): string | null {+ if (sub.has(Hidden)) return null;++ const value = store(world, Computed).value[sub.id()]!;+ const type = store(world, Effect).type[sub.id()] ?? 0;++ if (type === EffectType.LAYER_BLUR) {+ const clamped = Math.max(0, value);+ return clamped > EPSILON ? `blur(${clamped}px)` : null;+ }+ if (type === EffectType.BRIGHTNESS) {+ const clamped = Math.min(1, Math.max(0, value));+ return Math.abs(clamped - 1) > EPSILON ? `brightness(${clamped})` : null;+ }+ if (type === EffectType.CONTRAST) {+ const clamped = Math.min(1, Math.max(0, value));+ return Math.abs(clamped - 1) > EPSILON ? `contrast(${clamped})` : null;+ }+ if (type === EffectType.GRAYSCALE) {+ const clamped = Math.min(1, Math.max(0, value));+ return clamped > EPSILON ? `grayscale(${clamped})` : null;+ }+ if (type === EffectType.HUE_ROTATION) {+ return Math.abs(value) > EPSILON ? `hue-rotate(${value}deg)` : null;+ }+ if (type === EffectType.INVERT) {+ const clamped = Math.min(1, Math.max(0, value));+ return clamped > EPSILON ? `invert(${clamped})` : null;+ }+ if (type === EffectType.SATURATE) {+ const clamped = Math.min(1, Math.max(0, value));+ return Math.abs(clamped - 1) > EPSILON ? `saturate(${clamped})` : null;+ }+ if (type === EffectType.SEPIA) {+ const clamped = Math.min(1, Math.max(0, value));+ return clamped > EPSILON ? `sepia(${clamped})` : null;+ }+ return null;+}++/** CSS filter string from the entity's own blur plus effect sub-entities. */+function buildEffects(world: World, entity: Entity): string | null {+ const parts: string[] = [];++ const blurVal = store(world, Computed).blur[entity.id()]!;+ if (blurVal > EPSILON) {+ parts.push(`blur(${blurVal}px)`);+ }++ const effects = store(world, Cache).effects[entity.id()] ?? [];+ for (const effect of effects) {+ const f = effectFilter(world, effect);+ if (f) parts.push(f);+ }++ if (parts.length === 0) return null;++ return parts.join(' ');+}++export function renderFills(world: World, entity: Entity): void {+ const ctx = getCtx(world);+ const computed = store(world, Computed);+ const paintStore = store(world, Paint);+ const appearance = store(world, Appearance);+ const scaleMode = store(world, ScaleMode);+ const eid = entity.id();+ const fills = store(world, Cache).fills[eid] ?? [];++ for (let index = 0; index < fills.length; index++) {+ const fill = fills[index]!;+ if (fill.has(Hidden) || shaderConsumesFill(world, entity, fills, index)) continue;+ const fid = fill.id();+ const savedAlpha = ctx.globalAlpha;+ const savedCO = ctx.globalCompositeOperation;+ const bi = appearance.blendMode[fid] ?? 0;++ if (bi !== 0) {+ ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[bi]!;+ }++ ctx.globalAlpha = savedAlpha * computed.opacity[fid]!;++ const paint = paintStore.value[fid];+ if (paint === PaintType.IMAGE) {+ const decoder = resolveImageDecoder(world, fill)?.decoder;+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;+ const bitmap = decoder?.getBitmap(w, h);++ if (bitmap) {+ ctx.save();+ ctx.clip();++ const mode = scaleMode.value[fid] ?? 0;+ const [dx, dy, sw, sh] = getScaledImageProps(mode, bitmap.width, bitmap.height, w, h);+ ctx.drawImage(bitmap, dx, dy, sw, sh);++ ctx.restore();+ } else if (decoder?.failed) {+ ctx.fillStyle = MISSING_ASSET_COLOR;+ ctx.fill();+ }+ } else if (paint === PaintType.VIDEO) {+ const decoder = resolveVideoDecoder(world, fill);+ const frame = decoder?.toBitmap();++ if (frame) {+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;++ ctx.save();+ ctx.clip();++ const mode = scaleMode.value[fid] ?? 0;+ const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);+ ctx.drawImage(frame, dx, dy, sw, sh);++ ctx.restore();+ } else if (decoder?.errored) {+ ctx.fillStyle = MISSING_ASSET_COLOR;+ ctx.fill();+ }+ } else if (paint === PaintType.SEQUENCE) {+ const decoder = resolveSequenceDecoder(world, fill);+ const frame = decoder?.toBitmap();++ if (frame) {+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;++ ctx.save();+ ctx.clip();++ const mode = scaleMode.value[fid] ?? 0;+ const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);+ ctx.drawImage(frame, dx, dy, sw, sh);++ ctx.restore();+ } else if (decoder?.errored) {+ ctx.fillStyle = MISSING_ASSET_COLOR;+ ctx.fill();+ }+ } else if (paint === PaintType.HTML) {+ const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;++ if (host) {+ ctx.save();+ ctx.clip();+ host.draw(ctx, computed.width[eid]!, computed.height[eid]!);+ ctx.restore();+ }+ } else if (paint === PaintType.SURFACE) {+ const host = fill.has(SurfaceHostHandle) ? fill.get(SurfaceHostHandle) : null;++ if (host) {+ ctx.save();+ ctx.clip();+ host.draw(ctx, computed.width[eid]!, computed.height[eid]!);+ ctx.restore();+ }+ } else if (paint === PaintType.SOLID) {+ ctx.fillStyle = colorToHex(computed.color[fid]!);+ ctx.fill();+ } else if (paint === PaintType.LINEAR_GRADIENT) {+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;+ ctx.fillStyle = createLinearGradient(world, fill, ctx, w, h);+ ctx.fill();+ } else if (paint === PaintType.RADIAL_GRADIENT) {+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;+ ctx.fillStyle = createRadialGradient(world, fill, ctx, w, h);+ ctx.fill();+ } else if (paint === PaintType.WAVEFORM) {+ renderWaveform(world, entity, fill);+ } else if (paint === PaintType.SHADER) {+ renderShaderFill(world, entity, fills, index);+ }++ ctx.globalCompositeOperation = savedCO;+ ctx.globalAlpha = savedAlpha;+ }+}++/** The current frame of a video/image paint, as a GPU-uploadable source. */+function shaderSourceBitmap(+ world: World,+ fill: Entity,+ w: number,+ h: number,+): { source: GPUCopyExternalImageSource; width: number; height: number } | null {+ const paint = store(world, Paint).value[fill.id()];++ if (paint === PaintType.IMAGE) {+ const bitmap = resolveImageDecoder(world, fill)?.decoder?.getBitmap(w, h);+ return bitmap ? { source: bitmap, width: bitmap.width, height: bitmap.height } : null;+ }+ if (paint === PaintType.VIDEO) {+ const frame = resolveVideoDecoder(world, fill)?.toBitmap();+ return frame ? { source: frame, width: frame.width, height: frame.height } : null;+ }+ return null;+}++/**+ * Whether the fill at `index` is the input of a ready shader paint directly+ * above it. Only the immediate neighbor counts (a hidden paint in between+ * decouples the pair). Must mirror `renderShaderFill`'s input selection+ * exactly — a consumed media paint that the shader then fails to draw would+ * blank the element, so both sides check pipeline readiness and frame+ * availability.+ */+function shaderConsumesFill(world: World, entity: Entity, fills: Entity[], index: number): boolean {+ const paintStore = store(world, Paint);+ const computed = store(world, Computed);+ const fill = fills[index]!;+ const paint = paintStore.value[fill.id()];+ if (paint !== PaintType.IMAGE && paint !== PaintType.VIDEO) return false;++ const next = fills[index + 1];+ if (next === undefined || paintStore.value[next.id()] !== PaintType.SHADER) return false;+ if (next.has(Hidden)) return false;+ if (!resolveShaderHost(world, next)?.ready) return false;++ const w = computed.width[entity.id()]!;+ const h = computed.height[entity.id()]!;+ return shaderSourceBitmap(world, fill, w, h) !== null;+}++/**+ * Draws a shader paint: the media paint directly below it in the fill stack+ * is sampled as the shader's `source` texture and its output lands in the+ * parent's box in the media paint's place. Without a media paint below the+ * shader runs procedurally over a transparent source; before the pipeline is+ * ready it draws nothing and the media, if any, draws normally.+ */+function renderShaderFill(world: World, entity: Entity, fills: Entity[], index: number): void {+ const ctx = getCtx(world);+ const paintStore = store(world, Paint);+ const computed = store(world, Computed);++ const host = resolveShaderHost(world, fills[index]!);+ if (!host?.ready) return;++ const eid = entity.id();+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;++ // A visible media paint directly below is the shader's input. Anything+ // else (no fill below, a hidden one, a solid/gradient) runs the shader+ // procedurally over a transparent source, stacking like a normal paint.+ const media = fills[index - 1];+ const isMedia = media !== undefined+ && !media.has(Hidden)+ && (paintStore.value[media.id()] === PaintType.IMAGE || paintStore.value[media.id()] === PaintType.VIDEO);+ let input: ReturnType<typeof shaderSourceBitmap> = null;+ if (isMedia) {+ input = shaderSourceBitmap(world, media, w, h);+ if (!input) return;+ }++ const fit = input+ ? getScaledImageProps(store(world, ScaleMode).value[media!.id()] ?? 0, input.width, input.height, w, h)+ : [0, 0, w, h] as [number, number, number, number];+ const fps = world.get(FrameRate)?.value ?? 30;+ const time = (computed.localTime[eid] ?? 0) / fps;++ ctx.save();+ ctx.clip();+ host.draw(ctx, w, h, input?.source ?? null, input?.width ?? 1, input?.height ?? 1, fit, time, store(world, Shader).uniforms[fills[index]!.id()] ?? null);+ ctx.restore();+}++function renderShadows(world: World, entity: Entity): void {+ const ctx = getCtx(world);+ const computed = store(world, Computed);++ const shadows = store(world, Cache).shadows[entity.id()];+ if (!shadows) return;++ ctx.save();+ const savedAlpha = ctx.globalAlpha;++ // ctx.shadowBlur/OffsetX/OffsetY are in device-pixel space and are not+ // affected by the current transform, so scale them up to match the+ // content transform (camera * resolution).+ const camera = world.get(Camera);+ const resolution = world.get(RenderSurface)?.resolution ?? 1;+ const shadowScale = (camera?.a ?? 1) * resolution;++ for (const shadow of shadows) {+ if (shadow.has(Hidden)) continue;+ const sid = shadow.id();+ const color = colorToHex(computed.color[sid]!);+ ctx.shadowColor = color;+ ctx.fillStyle = color;+ ctx.globalAlpha = savedAlpha * computed.opacity[sid]!;+ ctx.shadowBlur = computed.blur[sid]! * shadowScale;+ ctx.shadowOffsetX = computed.offsetX[sid]! * shadowScale;+ ctx.shadowOffsetY = computed.offsetY[sid]! * shadowScale;+ ctx.fill();+ }++ ctx.restore();+}++function renderStrokes(world: World, entity: Entity): void {+ const ctx = getCtx(world);+ const eid = entity.id();+ const strokes = store(world, Cache).strokes[eid];+ if (!strokes) return;++ const strokeStyle = store(world, StrokeStyle);+ const computed = store(world, Computed);+ const appearance = store(world, Appearance);+ const paintStore = store(world, Paint);++ ctx.lineWidth = computed.strokeWidth[eid]!;+ ctx.lineJoin = StrokeJoin[strokeStyle.join[eid] ?? 0]!.toLocaleLowerCase() as CanvasLineJoin;+ ctx.lineCap = StrokeCap[strokeStyle.cap[eid] ?? 0]!.toLocaleLowerCase() as CanvasLineCap;+ ctx.miterLimit = strokeStyle.miterLimit[eid] ?? 3;++ for (const stroke of strokes) {+ if (stroke.has(Hidden)) continue;+ const sid = stroke.id();+ const savedAlpha = ctx.globalAlpha;+ const savedCO = ctx.globalCompositeOperation;+ const bi = appearance.blendMode[sid] ?? 0;++ if (bi !== 0) {+ ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[bi]!;+ }++ ctx.globalAlpha = savedAlpha * computed.opacity[sid]!;++ const paintType = paintStore.value[sid];+ if (paintType === PaintType.LINEAR_GRADIENT) {+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;+ ctx.strokeStyle = createLinearGradient(world, stroke, ctx, w, h);+ } else if (paintType === PaintType.RADIAL_GRADIENT) {+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;+ ctx.strokeStyle = createRadialGradient(world, stroke, ctx, w, h);+ } else {+ ctx.strokeStyle = colorToHex(computed.color[sid]!);+ }+ ctx.stroke();++ ctx.globalCompositeOperation = savedCO;+ ctx.globalAlpha = savedAlpha;+ }+}++/**+ * Pulse animation for entities with the Generating tag.+ *+ * Easing: cubic-bezier(0.52, 0.18, 0.56, 0.88)+ * Duration: 0.6s per half-cycle (alternating)+ * Delay: 0.2s before the first transition+ * Full cycle: 0.2 delay + 0.6 forward + 0.6 reverse = 1.4s+ * repeats ∞ for the entire gen phase (~4.5s)+ *+ * Pulses between --background (#1c1c1c) and --secondary (#292929).+ */+const generatingEase = cubicBezier(0.52, 0.18, 0.56, 0.88);++const GEN_DELAY = 200; // ms+const GEN_DURATION = 600; // ms per half-cycle+const GEN_CYCLE = GEN_DURATION * 2; // full forward+reverse cycle++// Colors: #1c1c1c → rgb(28,28,28) and #292929 → rgb(41,41,41)+const GEN_FROM_R = 28, GEN_FROM_G = 28, GEN_FROM_B = 28;+const GEN_TO_R = 41, GEN_TO_G = 41, GEN_TO_B = 41;++function renderGenerating(world: World, entity: Entity): void {+ const ctx = getCtx(world);+ if (!entity.has(Generating)) return;++ const t = world.get(Time)?.now ?? 0;++ // Before initial delay → show base color+ const elapsed = t % (GEN_DELAY + GEN_CYCLE);+ let factor = 0;++ if (elapsed >= GEN_DELAY) {+ const cycleT = elapsed - GEN_DELAY;+ const half = cycleT % GEN_DURATION;+ const progress = half / GEN_DURATION;+ const eased = generatingEase(progress);++ // Alternate direction: first half forward, second half reverse+ const inReverse = cycleT >= GEN_DURATION;+ factor = inReverse ? 1 - eased : eased;+ }++ const r = Math.round(GEN_FROM_R + (GEN_TO_R - GEN_FROM_R) * factor);+ const g = Math.round(GEN_FROM_G + (GEN_TO_G - GEN_FROM_G) * factor);+ const b = Math.round(GEN_FROM_B + (GEN_TO_B - GEN_FROM_B) * factor);++ ctx.fillStyle = `rgb(${r},${g},${b})`;+ ctx.fill();+}++// ── WAVEFORM paint ─────────────────────────────────────+//+// Renders an audio asset's pre-computed peaks as a bar chart inside the+// parent geometry's bounds. Sourced from the paint's own AssetId — the paint+// carries its asset reference, exactly like IMAGE/VIDEO/SEQUENCE paints.++const WAVEFORM_BAR_WIDTH = 6;+const WAVEFORM_BAR_GAP = 6;+const WAVEFORM_BAR_RADIUS = WAVEFORM_BAR_WIDTH / 2;+const WAVEFORM_MIN_BAR_HEIGHT = 4;+const WAVEFORM_PADDING = 12;+const WAVEFORM_BG_COLOR = '#202020';+const WAVEFORM_BG_RADIUS = 12;++function renderWaveform(world: World, entity: Entity, fill: Entity): void {+ const ctx = getCtx(world);+ const computed = store(world, Computed);++ const assetId = store(world, AssetId).value[fill.id()];+ if (!assetId) return;++ const asset = world.get(Assets)?.get(assetId);+ // Both AUDIO and VIDEO assets carry pre-computed peaks (video has audio).+ if (asset?.type !== 'AUDIO' && asset?.type !== 'VIDEO') return;++ const peaks = getAudioPeaks(asset);+ if (!peaks || peaks.length === 0) return;++ const w = computed.width[entity.id()]!;+ const h = computed.height[entity.id()]!;++ ctx.save();+ ctx.clip();++ // Background+ ctx.fillStyle = WAVEFORM_BG_COLOR;+ ctx.beginPath();+ ctx.roundRect(0, 0, w, h, WAVEFORM_BG_RADIUS);+ ctx.fill();++ // Bars+ const step = WAVEFORM_BAR_WIDTH + WAVEFORM_BAR_GAP;+ const availableWidth = w - WAVEFORM_PADDING * 2;+ const barCount = Math.floor(availableWidth / step);+ const maxBarHeight = h - WAVEFORM_PADDING * 2;+ if (barCount <= 0 || maxBarHeight <= 0) {+ ctx.restore();+ return;+ }++ const startX = WAVEFORM_PADDING + (availableWidth - barCount * step + WAVEFORM_BAR_GAP) / 2;++ ctx.fillStyle = '#ffffff';++ for (let i = 0; i < barCount; i++) {+ const peakIndex = Math.floor((i / barCount) * peaks.length);+ const value = (peaks[peakIndex] ?? 0) / 255;+ const barHeight = Math.max(value * maxBarHeight, WAVEFORM_MIN_BAR_HEIGHT);+ const x = startX + i * step;+ const y = (h - barHeight) / 2;++ ctx.beginPath();+ ctx.roundRect(x, y, WAVEFORM_BAR_WIDTH, barHeight, WAVEFORM_BAR_RADIUS);+ ctx.fill();+ }++ ctx.restore();+}++function renderShapeNode(world: World, entity: Entity): void {+ drawRectPath(world, entity);+ renderShadows(world, entity);+ renderFills(world, entity);+ renderGenerating(world, entity);+ renderStrokes(world, entity);+}++function renderTextNode(world: World, entity: Entity): void {+ renderText(world, entity);+}++function renderCaptionNode(world: World, entity: Entity): void {+ resolveCaptionDecoder(world, entity)?.draw(world, entity);+}++// ── Transition rendering ─────────────────────────────────────++function renderTransition(world: World, scene: Entity, left: Entity): void {+ const ctx = getCtx(world);+ const computed = store(world, Computed);++ const currentTime = computed.localTime[scene.id()]!;++ const children = store(world, Cache).children[scene.id()] ?? [];+ const right = children.find(sibling => computed.start[sibling.id()] === computed.end[left.id()]);+ if (!right) return;++ const win = getTransitionWindow(world, left, right);++ if (currentTime < win.start || currentTime >= win.end) return;++ // we are transitioning+ const duration = win.end - win.start;+ const completion = (currentTime - win.start) / duration;++ const type = store(world, Transition).type[left.id()] ?? TransitionType.DISSOLVE;++ const parent = getParentNode(left);+ if (parent === null) return;+ const width = computed.width[parent.id()]!;+ const height = computed.height[parent.id()]!;++ switch (type) {+ case TransitionType.SLIDE_FROM_RIGHT: {+ renderNode(world, left);+ ctx.save();+ ctx.translate(((1 - completion) ** 2 * width) | 0, 0);+ renderNode(world, right);+ ctx.restore();+ break;+ }+ case TransitionType.SLIDE_FROM_LEFT: {+ renderNode(world, left);+ ctx.save();+ ctx.translate(((1 - completion) ** 2 * width * -1) | 0, 0);+ renderNode(world, right);+ ctx.restore();+ break;+ }+ case TransitionType.FADE_TO_BLACK: {+ if (completion < 0.5) {+ renderNode(world, left);+ } else {+ renderNode(world, right);+ }+ ctx.save();+ ctx.beginPath();+ ctx.rect(0, 0, width, height);+ ctx.closePath();+ ctx.fillStyle = '#000000';+ ctx.globalAlpha = completion < 0.5 ? 2 * completion : 2 * (1 - completion);+ ctx.fill();+ ctx.restore();+ break;+ }+ case TransitionType.FADE_TO_WHITE: {+ if (completion < 0.5) {+ renderNode(world, left);+ } else {+ renderNode(world, right);+ }+ ctx.save();+ ctx.beginPath();+ ctx.rect(0, 0, width, height);+ ctx.closePath();+ ctx.fillStyle = '#FFFFFF';+ ctx.globalAlpha = completion < 0.5 ? 2 * completion : 2 * (1 - completion);+ ctx.fill();+ ctx.restore();+ break;+ }+ default: {+ // Dissolve (default)+ renderNode(world, left);+ ctx.save();+ ctx.globalAlpha = completion;+ renderNode(world, right);+ ctx.restore();+ break;+ }+ }++ // Mark both partners as already drawn this frame so the parent's+ // children loop skips its plain renderNode pass for them.+ computed.visibility[left.id()] = 0;+ computed.visibility[right.id()] = 0;+}++export function renderNode(world: World, entity: Entity): void {+ const ctx = getCtx(world);+ const computed = store(world, Computed);+ const eid = entity.id();++ if (computed.visibility[eid] === 0 || entity.has(Culled)) return;++ if (entity.has(Interactive)) {+ world.get(HitRegions)?.list.push({+ target: { kind: 'entity', id: entity },+ });+ }++ if (entity.has(IsMask) || entity.has(Hidden)) return;++ ctx.save();++ const local = store(world, LocalTransform);+ ctx.transform(+ local.a[eid]!,+ local.b[eid]!,+ local.c[eid]!,+ local.d[eid]!,+ local.e[eid]!,+ local.f[eid]!,+ );++ const worldTransform = store(world, WorldTransform);+ for (const mask of store(world, Cache).masks[eid] ?? []) {+ if (computed.visibility[mask.id()] === 0) continue;+ ctx.save();+ ctx.setTransform(+ worldTransform.a[mask.id()]!,+ worldTransform.b[mask.id()]!,+ worldTransform.c[mask.id()]!,+ worldTransform.d[mask.id()]!,+ worldTransform.e[mask.id()]!,+ worldTransform.f[mask.id()]!,+ );+ drawRectPath(world, mask);+ ctx.restore();+ ctx.clip();+ }++ // Appearance+ ctx.globalAlpha *= computed.opacity[eid]!;+ const bi = store(world, Appearance).blendMode[eid] ?? 0;+ if (bi !== 0) ctx.globalCompositeOperation = COMPOSITE_OPERATIONS[bi]!;++ const effects = buildEffects(world, entity);+ let initialFilter = 'none';++ if (effects !== null) {+ initialFilter = ctx.filter;+ ctx.filter = effects;+ }+++ if (entity.has(Caption)) {+ renderCaptionNode(world, entity);+ } else if (store(world, Geometry).value[eid] === GeometryType.TEXT) {+ renderTextNode(world, entity);+ } else if (store(world, Geometry).value[eid] === GeometryType.RECT) {+ renderShapeNode(world, entity);+ }++ // Clip and render children+ const children = store(world, Cache).children[eid] ?? [];+ if (children.length) {+ if (entity.has(ClipsContent)) {+ ctx.save();+ ctx.clip();+ }++ for (const child of children) {+ // Edge case: Child with transition+ if (child.has(Transition)) {+ renderTransition(world, entity, child);+ // Note: we are not breaking here since the transition handler will hide/unhide the children+ }++ renderNode(world, child);+ }++ if (entity.has(ClipsContent)) {+ ctx.restore();+ }+ }++ // Reset filter after drawing+ if (initialFilter !== 'none') {+ ctx.filter = initialFilter;+ }++ ctx.restore();+}++/**+ * Render system entry point. Call after transformSystem.+ *+ * Reads camera, background, and canvas size from world state and applies+ * DPR * Camera as the base canvas transform before drawing top-level nodes.+ * Without a render surface (headless world) this is a no-op.+ */+export function renderSystem(world: World): void {+ const surface = world.get(RenderSurface);+ const ctx = surface?.ctx;+ const canvas = surface?.canvas;+ if (!ctx || !canvas) return;++ const cw = canvas.width;+ const ch = canvas.height;++ // Clear + background (identity transform for full-canvas clear)+ ctx.setTransform(1, 0, 0, 1, 0, 0);+ ctx.clearRect(0, 0, cw, ch);++ // The stage background is a preview-only affordance; offline encoding+ // renders just the scene onto a transparent canvas (the scene paints its+ // own fill if it has one).+ if (world.get(Mode)?.value === 'realtime') {+ ctx.fillStyle = colorToHex(world.get(Background)?.value ?? 0);+ ctx.fillRect(0, 0, cw, ch);+ world.get(HitRegions)?.list.push({+ target: { kind: 'hud', id: 'canvas', quad: getCanvasQuad(cw, ch) },+ });+ }++ // Apply camera transform: DPR * Camera+ const camera = world.get(Camera) ?? { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 };+ const resolution = surface.resolution;+ ctx.setTransform(+ camera.a * resolution,+ camera.b * resolution,+ camera.c * resolution,+ camera.d * resolution,+ camera.e * resolution,+ camera.f * resolution,+ );++ // Render top-level nodes.+ const document = getDocument(world);+ for (const entity of world.query(Or(Geometry, Group), ChildOf(document), Not(Deleted), Not(Culled))) {+ renderNode(world, entity);+ }+}++function getCanvasQuad(width: number, height: number): Quad {+ return [+ { x: 0, y: 0 },+ { x: width, y: 0 },+ { x: width, y: height },+ { x: 0, y: height },+ ];+}diff --git a/packages/runtime/src/systems/schedule.ts b/packages/runtime/src/systems/schedule.tsindex 1f1213f5..10929567 100644--- a/packages/runtime/src/systems/schedule.ts+++ b/packages/runtime/src/systems/schedule.ts@@ -1 +1,22 @@-// System ordering for realtime vs capture mode.+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Deterministic system order for one frame, shared by the realtime tick and+// the capture loop (which sets Time/playheads explicitly, runs this, then+// awaits FramePromises before sampling). The app's tick wraps this with its+// browser-only systems (input before, HUD/dom sync after).++import { playbackSystem } from './playback';+import { motionSystem } from './motion';+import { transformSystem } from './transform';+import { renderSystem } from './render';++import type { World } from 'koota';++export function runSystems(world: World): void {+ playbackSystem(world);+ motionSystem(world);+ transformSystem(world);+ renderSystem(world);+}diff --git a/packages/runtime/src/traits/world.ts b/packages/runtime/src/traits/world.tsindex 45d1c63c..01c0e943 100644--- a/packages/runtime/src/traits/world.ts+++ b/packages/runtime/src/traits/world.ts@@ -8,6 +8,7 @@ import { createLiveMounts } from '../utils/live-mounts';import type { FontSource } from '../fonts/types';import type { Asset, Folder } from '../assets/types';+import type { Quad } from '../math/aabb';// Singleton state attached to the world itself (world.get/world.set).// Replaces the bitecs createWorld({...}) context bag. Only headless runtime@@ -84,3 +85,26 @@ export const Mounts = trait(() => createLiveMounts());export const FramePromises = trait({list: () => null as (Promise<unknown> | null)[] | null,});++export type EntityTarget = {+ kind: 'entity';+ id: Entity;+};++export type HudTarget = {+ kind: 'hud';+ id: string;+ quad: Quad;+};++// Paint-order hit regions collected during a render pass (topmost last). The+// render system pushes callback-less entries for Interactive entities and the+// stage canvas; the app's input system clears the list each frame, pushes its+// HUD controls with handlers attached, and maps callback-less targets to its+// default interaction handlers.+export type HitRegion = {+ target: EntityTarget | HudTarget;+ callback?: (world: unknown, event: unknown) => void;+};++export const HitRegions = trait({ list: () => [] as HitRegion[] });diff --git a/packages/runtime/src/utils/time.ts b/packages/runtime/src/utils/time.tsindex 41bc5ad2..ebc6698e 100644--- a/packages/runtime/src/utils/time.ts+++ b/packages/runtime/src/utils/time.ts@@ -26,6 +26,36 @@ export function framesToSeconds(frames: number = 0, fps: number = 30) {return snapToMs(frames / fps);}+function splitTimecode(seconds: number, frameRate: number): [hours: number, minutes: number, seconds: number, frames: number] {+ const fps = Math.max(1, Math.round(frameRate));+ const totalFrames = Math.round(seconds * fps);+ const totalSeconds = Math.floor(totalFrames / fps);+ return [+ Math.floor(totalSeconds / 3600),+ Math.floor((totalSeconds % 3600) / 60),+ totalSeconds % 60,+ totalFrames % fps,+ ];+}++/** `HH:MM:SS:FF`, fixed width so ruler ticks line up. */+export function formatTimestamp(seconds: number, frameRate: number): string {+ return splitTimecode(seconds, frameRate).map((v) => String(v).padStart(2, '0')).join(':');+}++/**+ * The same timecode with its zero segments dropped, for labels and filenames:+ * `08s10f`, `01m05s`, `0f`. Each segment carries its unit, so nothing is+ * ambiguous once the empty ones are gone.+ */+export function formatTimecode(seconds: number, frameRate: number): string {+ const units = ['h', 'm', 's', 'f'];+ const stamp = splitTimecode(seconds, frameRate)+ .map((value, i) => (value === 0 ? '' : `${String(value).padStart(2, '0')}${units[i]}`))+ .join('');+ return stamp || '0f';+}+/*** Computes the trim start frame (local) for a given entity,* based on the specified start frame (global) and playback rate.diff --git a/packages/runtime/src/world/create-world.ts b/packages/runtime/src/world/create-world.tsindex 01373fdb..f08db4b8 100644--- a/packages/runtime/src/world/create-world.ts+++ b/packages/runtime/src/world/create-world.ts@@ -22,6 +22,7 @@ import {AssetIds,Mounts,FramePromises,+ HitRegions,} from '../traits/world';/**@@ -47,6 +48,7 @@ export function createRuntimeWorld(projectId: string) {AssetIds,Mounts,FramePromises,+ HitRegions,);// The document root all rendered entities hang off (see DocumentRoot).