Skip to content

Code-first video & image, rendered natively on WebGPU.

6.1 KB
Raw Blame
/// <reference lib="webworker" />
/**
 * The preview's soundtrack: the export's mixer run in the browser, a stretch
 * at a time. The page asks a worker running this (with its own GPU device)
 * for the blocks around its playhead, so mixing never holds up drawing; it
 * mixes on the page itself only if the worker can't.
 */
import { BUILTIN_NODE_RENDERERS } from "framefields-preview:renderers";
import { mixAudioTracks, preflightMediaTime } from "@framefields/compositions";
import type { VirtualMediaData } from "@framefields/core";
import { ensureDevice } from "@framefields/webgpu-renderers";
import { registerRenderers } from "./register.js";

/** A stretch of the soundtrack, before the master stage. */
export interface PreviewMix {
	left: Float32Array;
	right: Float32Array;
	sampleRate: number;
	/** Its loudest sample: the page masters with the film's loudest. */
	peak: number;
	/**
	 * The same samples as that stretch of a full mix: mixed from the film's
	 * start, or continuing a chain of back-to-back stretches that began
	 * there. Otherwise effects with memory start from a lead-in.
	 */
	exact: boolean;
}

export interface MixRange {
	startSec: number;
	endSec: number;
}

/** What a composition export gives the mixer. */
export interface AudioTarget {
	toVirtualMediaAsync?: () => Promise<VirtualMediaData>;
	toVirtualMedia?: () => VirtualMediaData;
	/** Finishes live signal time, recorded frame by frame, before a mix. */
	prepareMediaTime?: () => Promise<void>;
}

/**
 * Seconds mixed before a window and dropped, so effects that carry state
 * (reverb tails, delays, compressors) enter it warmed up, as in a full mix.
 */
const AUDIO_LEAD_IN_SEC = 2;

/** The mixing function `createAudioMixer` returns. */
export type AudioMixer = (
	range: MixRange,
	chain?: string,
) => Promise<PreviewMix>;

/**
 * Mixes [startSec, endSec) of the target's soundtrack with the export's
 * mixer: the same decoders (reading only the stretch it plays, so pinned
 * assets stream from their bucket), audio effects on `device` and signals.
 *
 * Each `chain` keeps its own effect state. A stretch that starts where the
 * chain's last one ended continues that state, so a chain of back-to-back
 * stretches is the full mix, sample for sample; any other starts fresh from
 * a lead-in. The page plays on one chain and fills the film in on another,
 * so neither breaks the other.
 */
export function createAudioMixer(
	target: AudioTarget,
	fps: number,
	device: GPUDevice,
	renderId: string,
): AudioMixer {
	const chains = new Map<string, { endSec: number; exact: boolean }>();
	// Prepared once, as the export prepares it: a fresh tree, live signal
	// time finished, media timing preflighted.
	let prepared: Promise<VirtualMediaData> | undefined;
	const prepare = async () => {
		const tree = target.toVirtualMediaAsync
			? await target.toVirtualMediaAsync()
			: target.toVirtualMedia?.();
		if (!tree) throw new Error("The export is not a composition.");
		await target.prepareMediaTime?.();
		await preflightMediaTime(tree, fps);
		return tree;
	};
	return async (range, chain = "main") => {
		prepared ??= prepare();
		const tree = await prepared;
		const last = chains.get(chain);
		chains.delete(chain); // until this stretch is mixed
		const contiguous =
			last !== undefined && Math.abs(range.startSec - last.endSec) < 1e-6;
		const from = contiguous
			? range.startSec
			: Math.max(0, range.startSec - AUDIO_LEAD_IN_SEC);
		const exact = contiguous ? last.exact : from === 0;
		const { channels, sampleRate } = await mixAudioTracks(
			tree,
			fps,
			48_000,
			device,
			`${renderId}-${chain}`,
			{
				window: { startSec: from, endSec: range.endSec },
				master: false,
				continue: contiguous,
			},
		);
		chains.set(chain, { endSec: range.endSec, exact });
		const drop = Math.round((range.startSec - from) * sampleRate);
		const left = (channels[0] ?? new Float32Array(0)).slice(drop);
		const right = (channels[1] ?? channels[0] ?? left).slice(drop);
		let peak = 0;
		for (let i = 0; i < left.length; i++)
			peak = Math.max(peak, Math.abs(left[i]), Math.abs(right[i]));
		return { left, right, sampleRate, peak, exact };
	};
}

export interface MixRequest {
	id: number;
	range: MixRange;
	/** The effect-state chain to mix on (see `createAudioMixer`). */
	chain: string;
}

export type MixReply =
	| { ready: true }
	| { ready: false; error: string }
	| { id: number; mix: PreviewMix }
	| { id: number; error: string };

/**
 * The worker side: the project's bundle calls this instead of starting a
 * player when it runs in a worker. It builds the composition, then answers
 * each `MixRequest` with the stretch mixed (its buffers transferred).
 */
export function startAudioMixer(
	mod: Record<string, unknown>,
	exportName: string,
): void {
	const scope = self as unknown as DedicatedWorkerGlobalScope;
	const ready = (async () => {
		await registerRenderers(BUILTIN_NODE_RENDERERS);
		const exported = mod[exportName];
		const target = (
			typeof exported === "function"
				? await (exported as () => unknown)()
				: exported
		) as AudioTarget & { fps?: number };
		const vm = target.toVirtualMediaAsync
			? await target.toVirtualMediaAsync()
			: target.toVirtualMedia?.();
		const fps =
			target.fps ?? (vm?.operation as { fps?: number } | undefined)?.fps ?? 30;
		return createAudioMixer(target, fps, await ensureDevice(), "worker-audio");
	})();
	ready.then(
		() => scope.postMessage({ ready: true } satisfies MixReply),
		(err: unknown) =>
			scope.postMessage({
				ready: false,
				error: errorText(err),
			} satisfies MixReply),
	);
	// One stretch at a time, in the order asked: the page queues them.
	let queue: Promise<void> = Promise.resolve();
	scope.onmessage = (event: MessageEvent<MixRequest>) => {
		const { id, range, chain } = event.data;
		queue = queue.then(async () => {
			try {
				const mix = await (await ready)(range, chain);
				scope.postMessage({ id, mix } satisfies MixReply, [
					mix.left.buffer,
					mix.right.buffer,
				]);
			} catch (err) {
				scope.postMessage({ id, error: errorText(err) } satisfies MixReply);
			}
		});
	};
}

function errorText(err: unknown): string {
	return err instanceof Error ? err.message : String(err);
}