Skip to content

The VS Code of video editing, built for agents.

TypeScript
14.6 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import {
	CanvasSource,
	Output,
	AudioSample,
	AudioSampleSource,
	canEncodeVideo,
} from 'mediabunny';
import { Not } from 'koota';
import {
	setActive, framesToSeconds,
	assert, store, isScene,
	assetSystem, playbackSystem, motionSystem, transformSystem, renderSystem,
	AudioBus, AudioBusHandle,
	ChildOf, Geometry, Paint, Workarea, Playback,
	AudioPlayback, Computed,
	Position, Offset, Rotation, Scale, Skew,
	Time, FrameRate, RenderSurface, AudioEngine, Root,
	FramePromises,
} from '@diffusionstudio/runtime';

import { TargetBuffer } from './buffer';
import { createOutputFormat } from './format';
import { computeOutputSize, createRenderEventDetail } from './utils';

import type { Entity, World } from 'koota';
import type { EncoderConfig } from './interfaces';
import type { ExportResult } from './types';

/**
 * The scene a capture world holds. An encoder takes the world as it is —
 * what to encode is the whole of what is in it — so a stage with anything
 * else under it is a caller that has not finished preparing the world.
 */
export function captureScene(world: World): Entity {
	const roots = [...world.query(ChildOf(world.get(Root)!))];
	assert(roots.length === 1, `A capture world holds one scene, this one holds ${roots.length}`);

	const scene = roots[0]!;
	assert(isScene(scene), 'What the capture world holds is not a scene');
	return scene;
}

/**
 * Encodes the scene a capture world holds into a file.
 *
 * `world` is not the editor's: it is a world built for this encode, holding
 * the composition as it is to be written and nothing else — see the app's
 * capture builder, which renders the project into a fresh world and reduces
 * its stage to the one scene. Everything the encode needs is read from there:
 * the scene is the stage's only child, the frame rate is the world's, the
 * camera is the view it is drawn with, and the canvas the DOM-backed paints
 * attached themselves to is the one drawn into.
 *
 * The world stays the caller's — it built it, it disposes it. What is created
 * here is the audio context, the output, and the frames.
 */
export async function createEncoder(world: World, config: EncoderConfig) {
	const scene = captureScene(world);
	const sceneId = scene.id();
	const computed = store(world, Computed);
	const playback = store(world, Playback);

	await warmupAssets(world);

	const frameRate = world.get(FrameRate)?.value ?? 30;
	const sceneWidth = computed.width[sceneId]!;
	const sceneHeight = computed.height[sceneId]!;
	const sceneEnd = computed.end[sceneId]!;

	// Honor the scene's Workarea trait: render only the frames between
	// Workarea.start and Workarea.end instead of the full scene duration.
	const workarea = scene.get(Workarea);
	const workareaStart = workarea ? Math.max(0, Math.min(sceneEnd, workarea.start)) : 0;
	const workareaEnd = workarea
		? Math.max(workareaStart, Math.min(sceneEnd, workarea.end || sceneEnd))
		: sceneEnd;

	const playheadStartSeconds = framesToSeconds(workareaStart, frameRate);
	const duration = framesToSeconds(workareaEnd - workareaStart, frameRate);

	// Before the two below read them: ogg is an audio container, so asking for
	// one is asking for the audio alone — and with `videoEnabled` derived
	// first, it would still have gone looking for pictures to put in it.
	if (config.format === 'ogg') {
		config.audio = { ...config.audio, enabled: true };
		config.video = { ...config.video, enabled: false };
	}

	const audioEnabled = config.audio?.enabled ?? true;
	const videoEnabled = config.video?.enabled ?? true;
	const numberOfChannels = config.audio?.numberOfChannels ?? 2;
	const sampleRate = config.audio?.sampleRate ?? 48000;
	const audioBitrate = config.audio?.bitrate ?? 128e3;
	const videoBitrate = config.video?.bitrate ?? 10e6;
	const audioCodec = config.audio?.codec ?? 'aac';
	const videoCodec = config.video?.codec ?? 'avc';
	const containerFormat = config.format ?? 'mp4';
	const resolution = config.video?.resolution ?? 1080;
	const { scale, width, height } = computeOutputSize(sceneWidth, sceneHeight, resolution);
	const frameDuration = 1 / frameRate;

	// Fail before anything is written: mediabunny would reject the config on
	// the first frame anyway, with a message in codec strings; this one names
	// the size the resolution setting worked out to, which is what the user
	// can change.
	if (videoEnabled) {
		const encodable = await canEncodeVideo(videoCodec, { width, height, bitrate: videoBitrate });
		assert(
			encodable,
			`This browser cannot encode ${videoCodec.toUpperCase()} at ${width}×${height}. ` +
			'Choose a lower resolution, a lower bitrate, or another codec.',
		);
	}

	const canvas = world.get(RenderSurface)?.canvas;
	assert(canvas instanceof HTMLCanvasElement, 'The capture world has no canvas to draw into');
	canvas.width = width;
	canvas.height = height;
	world.set(RenderSurface, { resolution: scale });

	const offlineAudioCtx = new OfflineAudioContext(
		numberOfChannels,
		Math.ceil(duration * sampleRate),
		sampleRate,
	);
	world.set(AudioEngine, { context: offlineAudioCtx });

	setActive(world, scene);

	computed.localTimeInSeconds[sceneId] = playheadStartSeconds;
	computed.localTime[sceneId] = workareaStart;
	playback.playing[sceneId] = true;
	playback.loop[sceneId] = false;
	playback.speed[sceneId] = 1;
	// Anchor audio scheduling: context starts at t=0 but the playhead starts at
	// playheadStartSeconds, so audioDelay = -playheadStartSeconds shifts each
	// entity's scheduled audio back into the encoded window.
	if (!scene.has(AudioPlayback)) scene.add(AudioPlayback);
	const audioPlayback = store(world, AudioPlayback);
	audioPlayback.contextOffsetInSeconds[sceneId] = 0;
	audioPlayback.timelineOffsetInSeconds[sceneId] = playheadStartSeconds;

	// Set up mediabunny output
	const buffer = await TargetBuffer.create(config.target);
	const format = await createOutputFormat(buffer, config.format);
	const output = new Output({ format, target: buffer.target });

	if (config.comment) {
		output.setMetadataTags({ comment: config.comment });
	}

	const audioSource = new AudioSampleSource({
		codec: audioCodec,
		bitrate: audioBitrate,
	});

	const sceneFills = [...world.query(Paint, Not(Geometry), ChildOf(scene))];

	const videoSource = new CanvasSource(canvas, {
		codec: videoCodec,
		bitrate: videoBitrate,
		latencyMode: 'quality',
		alpha: containerFormat === 'webm' && !sceneFills.length
			? 'keep'
			: 'discard',
	});

	if (audioEnabled) {
		output.addAudioTrack(audioSource);
	}

	if (videoEnabled) {
		output.addVideoTrack(videoSource);
	}

	// Audio worklet streams render-quantum chunks back to the main thread so
	// we can hand them to mediabunny in step with the visual frame loop.
	const audioWorkletUrl = createAudioWorkletUrl();
	await offlineAudioCtx.audioWorklet.addModule(audioWorkletUrl);

	const sharedBuffer = new SharedArrayBuffer(4);
	const sharedUint32Array = new Uint32Array(sharedBuffer);
	const mixNode = offlineAudioCtx.createGain();

	const sinkNode = new AudioWorkletNode(offlineAudioCtx, 'sink', {
		channelCount: numberOfChannels,
		channelCountMode: 'explicit',
		numberOfInputs: 1,
		numberOfOutputs: 1,
		outputChannelCount: [numberOfChannels],
		processorOptions: {
			buffer: sharedUint32Array,
		},
	});

	mixNode.connect(sinkNode);
	sinkNode.connect(offlineAudioCtx.destination);

	// connect the scene bus to the mix node
	const sceneBus = new AudioBus(world, scene);
	scene.add(AudioBusHandle);
	scene.set(AudioBusHandle, sceneBus);
	sceneBus.connect(mixNode);

	let sampleIndex = 0;
	let audioSourcePromise: Promise<void> | null = null;

	const allAudioReceived = Promise.withResolvers<void>();

	sinkNode.port.onmessage = event => {
		if (output?.state === 'canceled') {
			return;
		}

		const planarBuffer = event.data as Float32Array;

		if (planarBuffer.length === 0) {
			allAudioReceived.resolve();
			return;
		}

		const sampleCount = planarBuffer.length / numberOfChannels;

		const audioSample = new AudioSample({
			data: planarBuffer,
			format: 'f32-planar',
			numberOfChannels,
			sampleRate,
			timestamp: sampleIndex / sampleRate,
		});
		audioSourcePromise = audioSource.add(audioSample).finally(() => {
			audioSample.close();
		});

		sampleIndex += sampleCount;
	};

	const emitProgress = createThrottledCallback(config.onProgress, 1000 / 3); // 3 times per second

	let canceled = false;
	const cancel = () => (canceled = true);

	const render = async (): Promise<ExportResult> => {
		try {
			await output.start();
			const start = performance.now();
			const startTime = start;
			const totalFrames = Math.floor(duration * frameRate);

			let audioRenderingDone = false;
			let audioRenderingCompleted: Promise<AudioBuffer> | null = null;
			if (audioEnabled) {
				audioRenderingCompleted = offlineAudioCtx.startRendering();
				audioRenderingCompleted.then(() => { audioRenderingDone = true; });
			}

			let lastAudioSampleCount = 0;

			for (let frame = 0; frame < totalFrames; frame++) {
				if (canceled) {
					await output.cancel();
					Atomics.store(sharedUint32Array, 0, 2 ** 31 - 1);
					return { type: 'canceled' };
				}

				const timestamp = frame * frameDuration;
				const playheadSeconds = playheadStartSeconds + timestamp;

				// advancePlayhead is a no-op in offline mode; the encoder owns the
				// playhead and writes both seconds + frames before each tick.
				computed.localTimeInSeconds[sceneId] = playheadSeconds;
				computed.localTime[sceneId] = Math.round(playheadSeconds * frameRate);
				const time = world.get(Time)!;
				world.set(Time, { delta: frameDuration * 1000, now: time.now + frameDuration * 1000 });

				{
					assetSystem(world);
					playbackSystem(world);
					await resolverSystem(world);
					motionSystem(world);
					// May be changed by a system
					normalizeSceneTransform(world, sceneId);
				}

				if (videoEnabled) {
					// only visual systems
					transformSystem(world);
					renderSystem(world);
				}

				if (videoEnabled) {
					await videoSource.add(timestamp, frameDuration);
				}

				if (audioEnabled) {
					await audioSourcePromise;

					while (Atomics.load(sharedUint32Array, 0) > sampleRate && !audioRenderingDone) {
						await new Promise(resolve => setTimeout(resolve, 0));
					}

					const totalAudioSampleCount = Math.floor(timestamp * sampleRate);
					const diff = totalAudioSampleCount - lastAudioSampleCount;
					Atomics.add(sharedUint32Array, 0, diff);
					lastAudioSampleCount = totalAudioSampleCount;
				}

				emitProgress(createRenderEventDetail(frame, totalFrames, startTime));
			}

			console.info(
				`Encoded frames at ${((totalFrames * 1000) / (performance.now() - start)).toFixed(2)}FPS`
			);

			if (audioEnabled) {
				assert(audioRenderingCompleted !== null, 'Audio rendering was not started');

				Atomics.add(
					sharedUint32Array,
					0,
					Math.ceil(duration * sampleRate) + sampleRate - lastAudioSampleCount,
				);

				await audioRenderingCompleted;

				sinkNode.port.postMessage(null);
				await allAudioReceived.promise;
			}

			console.info('Finalizing file');

			await output.finalize();

			console.info('Export complete');

			return {
				type: 'success',
				data: await buffer.close(containerFormat),
			};
		} catch (e) {
			return {
				type: 'error',
				error: e instanceof Error ? e : new Error('Unknown error'),
			};
		} finally {
			// The world, its canvas and the decoders opened in it are the
			// caller's to release; only the worklet is this encode's.
			URL.revokeObjectURL(audioWorkletUrl);
		}
	};

	return {
		render,
		cancel,
	};
}


function audioWorkletCode() {
	// eslint-disable-next-line no-undef
	class SinkProcessor extends AudioWorkletProcessor {
		buffer: Uint32Array;

		constructor(options: {
			processorOptions: {
				buffer: Uint32Array;
			};
		}) {
			super();

			this.buffer = options.processorOptions.buffer;

			this.port.onmessage = () => {
				this.port.postMessage(new Float32Array(0));
			};
		}

		process(inputs: Float32Array[][], outputs: Float32Array[][]) {
			const input = inputs[0];
			const output = outputs[0];
			const renderQuantum = output[0].length;

			while (Atomics.load(this.buffer, 0) < renderQuantum);

			const planarBuffer = new Float32Array(output.length * output[0].length);
			for (let channel = 0; channel < Math.min(input.length, output.length); channel++) {
				planarBuffer.set(input[channel], channel * output[0].length);
			}

			this.port.postMessage(planarBuffer, [planarBuffer.buffer]);

			Atomics.sub(this.buffer, 0, renderQuantum);

			return true;
		}
	}

	// eslint-disable-next-line no-undef
	registerProcessor('sink', SinkProcessor);
}

function createAudioWorkletUrl(): string {
	const blob = new Blob([`(${audioWorkletCode.toString()})()`], { type: 'application/javascript' });
	return URL.createObjectURL(blob);
}

function createThrottledCallback<T>(callback: ((value: T) => void) | undefined, intervalMs: number): (value: T) => void {
	let lastCalledAt = -Infinity;

	return (value: T) => {
		const now = performance.now();
		if (now - lastCalledAt < intervalMs) {
			return;
		}

		lastCalledAt = now;
		callback?.(value);
	};
}

export function normalizeSceneTransform(world: World, sceneId: number): void {
	const position = store(world, Position);
	const offset = store(world, Offset);
	const rotation = store(world, Rotation);
	const scaleStore = store(world, Scale);
	const skew = store(world, Skew);
	position.x[sceneId] = 0;
	position.y[sceneId] = 0;
	offset.x[sceneId] = 0;
	offset.y[sceneId] = 0;
	rotation.value[sceneId] = 0;
	scaleStore.x[sceneId] = 1;
	scaleStore.y[sceneId] = 1;
	skew.x[sceneId] = 0;
	skew.y[sceneId] = 0;
	const computed = store(world, Computed);
	computed.positionX[sceneId] = 0;
	computed.positionY[sceneId] = 0;
	computed.offsetX[sceneId] = 0;
	computed.offsetY[sceneId] = 0;
	computed.rotation[sceneId] = 0;
	computed.scaleX[sceneId] = 1;
	computed.scaleY[sceneId] = 1;
	computed.skewX[sceneId] = 0;
	computed.skewY[sceneId] = 0;
}

/**
 * Runs the asset system until it has nothing left in flight: each pass can
 * start resolutions the previous one's answers asked for (a modifier chain,
 * a transcript that waits for the scene's own sources to land). Bounded, so
 * a source that keeps re-requesting cannot hold an export open forever.
 */
export async function warmupAssets(world: World): Promise<void> {
	for (let pass = 0; pass < 16; pass++) {
		assetSystem(world);
		if (!world.get(FramePromises)?.list?.length) return;
		await resolverSystem(world);
	}
}

export async function resolverSystem(world: World) {
	const promises = world.get(FramePromises)?.list;
	if (promises?.length) {
		await Promise.all(promises.filter(promise => promise !== null));
		world.set(FramePromises, { list: [] });
	}
}