Skip to content

Code-first video & image, rendered natively on WebGPU.

39.5 KB
Raw Blame
import {
	getActiveMediaMetadata,
	getMediaType,
	mediaSourceIsHold,
	mediaSourceRange,
	mediaSourceTime,
	type ResolvedMediaTimeRemap,
	type VirtualMediaData,
} from "@framefields/core";
import { type AudioProcessor, audioRegistry } from "@framefields/node-sdk";
import {
	inputStore,
	resolveInputUrl,
	shaderStore,
	WebGPUAudioProcessor,
} from "@framefields/webgpu-renderers";
import { AudioSampleSink } from "mediabunny";
import { computeRenderParams } from "./apply-operations.js";
import { normalizeTimeline } from "./normalization.js";

// Progress chatter (every decode step) is debug output: shown only with LOG_LEVEL=debug or trace.
const verbose =
	typeof process !== "undefined" &&
	/^(debug|trace)$/i.test(process.env?.LOG_LEVEL ?? "");

const mediaLogger = {
	debug: (msg: unknown, ...args: unknown[]) => {
		if (!verbose) return;
		if (typeof msg === "object" && msg !== null) {
			console.info("[AudioExtractor]", msg, ...args);
		} else {
			console.info(`[AudioExtractor] ${msg}`, ...args);
		}
	},
	warn: (msg: unknown, ...args: unknown[]) => {
		if (typeof msg === "object" && msg !== null) {
			console.warn("[AudioExtractor]", msg, ...args);
		} else {
			console.warn(`[AudioExtractor] ${msg}`, ...args);
		}
	},
	error: (msg: unknown, ...args: unknown[]) => {
		if (typeof msg === "object" && msg !== null) {
			console.error("[AudioExtractor]", msg, ...args);
		} else {
			console.error(`[AudioExtractor] ${msg}`, ...args);
		}
	},
};

const yieldToMain = (): Promise<void> => {
	if (typeof window === "undefined") {
		return Promise.resolve();
	}
	const sched = (globalThis as Record<string, unknown>).scheduler as
		| { yield?: () => Promise<void> }
		| undefined;
	return typeof sched?.yield === "function"
		? sched.yield()
		: new Promise<void>((resolve) => setTimeout(resolve, 0));
};

const waitForDelayRender = async (timeoutMs = 30_000): Promise<void> => {
	await new Promise((resolve) => setTimeout(resolve, 100));

	const start = Date.now();
	return new Promise<void>((resolve) => {
		const check = () => {
			const browserHandles: unknown[] =
				typeof window !== "undefined" &&
				(window as unknown as { renderer_delayRenderHandles?: unknown[] })
					.renderer_delayRenderHandles
					? (
							window as unknown as {
								renderer_delayRenderHandles: unknown[];
							}
						).renderer_delayRenderHandles
					: [];

			const globalDelays =
				typeof globalThis !== "undefined"
					? (globalThis as any).__GATEWAI_DELAYS__
					: null;

			const hasBrowserDelays = browserHandles.length > 0;
			const hasGlobalDelays =
				globalDelays instanceof Set && globalDelays.size > 0;

			if (!hasBrowserDelays && !hasGlobalDelays) {
				resolve();
				return;
			}

			if (Date.now() - start > timeoutMs) {
				mediaLogger.warn(
					`[AudioExtractor] waitForDelayRender timed out waiting for handles: ${JSON.stringify(
						{
							browserHandles,
							globalDelays: globalDelays ? Array.from(globalDelays) : [],
						},
					)}`,
				);
				resolve();
				return;
			}

			setTimeout(check, 50);
		};
		check();
	});
};

export interface AudioClip {
	sourceUrl: string;
	destStartFrame: number;
	trimStartSec: number;
	durationFrames: number;
	speed: number;
	volume: number;
}

/**
 * Decodes all audio samples from a given source URL into raw Float32 mono/stereo channel buffers.
 */
export async function decodeAudioSource(
	url: string,
): Promise<{ channels: Float32Array[]; sampleRate: number } | null> {
	const decoded = await decodeAudioRange(url);
	return decoded
		? { channels: decoded.channels, sampleRate: decoded.sampleRate }
		: null;
}

/**
 * Seconds decoded ahead of a requested range: a compressed stream (MP3, AAC)
 * decoded from mid-file only matches a whole-file decode once the decoder has
 * warmed up on the packets before it.
 */
const DECODE_PREROLL_SEC = 0.5;

/**
 * A decoded stretch of a source. `channels[c][i]` is sample `startIndex + i`
 * of a whole-file decode, so callers index it as they would the whole file.
 * Samples in [validFromSec, validToSec) match a whole-file decode exactly.
 */
interface DecodedAudioRange {
	channels: Float32Array[];
	sampleRate: number;
	startIndex: number;
	validFromSec: number;
	validToSec: number;
}

/**
 * A source this runtime can't decode at all (a codec WebCodecs lacks, in a
 * worker without AudioContext). Thrown, not mixed as silence, so the caller
 * can mix where it can be decoded.
 */
export class AudioDecodeUnavailable extends Error {
	override name = "AudioDecodeUnavailable";
}

/**
 * The browser's whole-file decoder, for codecs WebCodecs can't decode
 * (mediabunny reads byte ranges and decodes with WebCodecs first).
 */
async function decodeWithAudioContext(url: string): Promise<DecodedAudioRange> {
	const AudioContextClass =
		window.AudioContext ||
		(window as Window & { webkitAudioContext?: typeof AudioContext })
			.webkitAudioContext;
	if (!AudioContextClass) {
		throw new AudioDecodeUnavailable(
			`No decoder here for ${url}: WebCodecs can't decode it and there is no AudioContext (a worker).`,
		);
	}
	const audioCtx = new AudioContextClass();
	try {
		// Local paths load through the preview server, as the input store does.
		const response = await fetch(resolveInputUrl(url, false));
		mediaLogger.debug(
			`[AudioExtractor] Fetch response status for ${url}: ${response.status} (${response.statusText})`,
		);
		if (!response.ok) {
			throw new Error(`Failed to fetch audio source: ${response.statusText}`);
		}
		const arrayBuffer = await response.arrayBuffer();
		mediaLogger.debug(
			`[AudioExtractor] Fetched ${arrayBuffer.byteLength} bytes. Decoding audio data...`,
		);
		const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer);
		mediaLogger.debug(
			`[AudioExtractor] Decoded successfully: ${audioBuffer.numberOfChannels} channels, sampleRate: ${audioBuffer.sampleRate}, duration: ${audioBuffer.duration}s`,
		);

		const channels: Float32Array[] = [];
		for (let c = 0; c < audioBuffer.numberOfChannels; c++) {
			channels.push(new Float32Array(audioBuffer.getChannelData(c)));
		}
		// decodeAudioData only decodes whole files.
		return {
			channels,
			sampleRate: audioBuffer.sampleRate,
			startIndex: 0,
			validFromSec: 0,
			validToSec: Number.POSITIVE_INFINITY,
		};
	} finally {
		void audioCtx.close();
		mediaLogger.debug(`[AudioExtractor] AudioContext closed for ${url}`);
	}
}

/**
 * Decodes [range.startSec, range.endSec) of a source (the whole file without
 * a range), so a short clip from a long file decodes only what it plays.
 */
async function decodeAudioRange(
	url: string,
	range?: { startSec: number; endSec: number },
): Promise<DecodedAudioRange | null> {
	mediaLogger.debug(
		`[AudioExtractor] decodeAudioSource started for url: ${url}`,
	);
	try {
		// A page or a worker: anything but Node.
		const isBrowser = !(
			typeof process !== "undefined" && process.versions?.node
		);
		let inputAcquired = false;
		try {
			mediaLogger.debug(
				`[AudioExtractor] Acquiring from inputStore for ${url}...`,
			);
			const input = await inputStore.acquire(url);
			inputAcquired = true;
			mediaLogger.debug(
				`[AudioExtractor] inputStore acquired. Retrieving primary audio track...`,
			);
			const track = await input.getPrimaryAudioTrack();
			if (!track) {
				mediaLogger.warn(
					`[AudioExtractor] No primary audio track found in input for ${url}`,
				);
				return null;
			}

			const sink = new AudioSampleSink(track);
			const decodeFromSec = range
				? Math.max(0, range.startSec - DECODE_PREROLL_SEC)
				: 0;
			const generator = sink.samples(decodeFromSec, range?.endSec);

			const decodedPlanes: Float32Array[][] = [];
			let totalFrames = 0;
			let sampleRate = 48000;
			let numChannels = 1;
			let firstTimestamp: number | null = null;

			mediaLogger.debug(
				`[AudioExtractor] Starting mediabunny generator loop for ${url}...`,
			);
			for await (const sample of generator) {
				if (firstTimestamp === null) firstTimestamp = sample.timestamp;
				sampleRate = sample.sampleRate;
				numChannels = sample.numberOfChannels;
				const sampleFrames = sample.numberOfFrames;

				if (decodedPlanes.length === 0) {
					for (let c = 0; c < numChannels; c++) {
						decodedPlanes.push([]);
					}
				}

				for (let c = 0; c < numChannels; c++) {
					let planeBuffer: Float32Array;
					try {
						const allocSize = sample.allocationSize({
							planeIndex: c,
							format: "f32-planar",
						});
						planeBuffer = new Float32Array(allocSize / 4);
						sample.copyTo(planeBuffer, {
							planeIndex: c,
							format: "f32-planar",
						});
					} catch {
						// Fallback: copy in sample's native format and normalize to float32 [-1, 1]
						const allocSize = sample.allocationSize({ planeIndex: c });
						if (sample.format === "s16" || sample.format === "s16-planar") {
							const int16Buffer = new Int16Array(allocSize / 2);
							sample.copyTo(int16Buffer, { planeIndex: c });
							planeBuffer = new Float32Array(int16Buffer.length);
							for (let i = 0; i < int16Buffer.length; i++) {
								planeBuffer[i] = (int16Buffer[i] ?? 0) / 32768;
							}
						} else if (
							sample.format === "s32" ||
							sample.format === "s32-planar"
						) {
							const int32Buffer = new Int32Array(allocSize / 4);
							sample.copyTo(int32Buffer, { planeIndex: c });
							planeBuffer = new Float32Array(int32Buffer.length);
							for (let i = 0; i < int32Buffer.length; i++) {
								planeBuffer[i] = (int32Buffer[i] ?? 0) / 2147483648;
							}
						} else if (
							sample.format === "u8" ||
							sample.format === "u8-planar"
						) {
							const u8Buffer = new Uint8Array(allocSize);
							sample.copyTo(u8Buffer, { planeIndex: c });
							planeBuffer = new Float32Array(u8Buffer.length);
							for (let i = 0; i < u8Buffer.length; i++) {
								planeBuffer[i] = ((u8Buffer[i] ?? 128) - 128) / 128;
							}
						} else {
							planeBuffer = new Float32Array(allocSize / 4);
							sample.copyTo(planeBuffer, { planeIndex: c });
						}
					}
					decodedPlanes[c].push(planeBuffer);
				}

				totalFrames += sampleFrames;
				sample.close();
			}
			mediaLogger.debug(
				`[AudioExtractor] Generator loop finished. Decoded total frames: ${totalFrames}`,
			);

			if (totalFrames === 0 || decodedPlanes.length === 0) {
				return null;
			}

			const channels = decodedPlanes.map((planes) => {
				const merged = new Float32Array(totalFrames);
				let offset = 0;
				for (const plane of planes) {
					merged.set(plane, offset);
					offset += plane.length;
				}
				return merged;
			});

			// A whole-file decode starts at the sample samples(0) yields first;
			// a range is placed on that same index.
			let startIndex = 0;
			if (decodeFromSec > 0 && firstTimestamp !== null) {
				const origin = await sink.getSample(0);
				const originTimestamp =
					origin?.timestamp ?? (await track.getFirstTimestamp());
				origin?.close();
				startIndex = Math.round(
					(firstTimestamp - originTimestamp) * sampleRate,
				);
			}
			const endedEarly =
				!range || (startIndex + totalFrames) / sampleRate < range.endSec;

			return {
				channels,
				sampleRate,
				startIndex,
				validFromSec: decodeFromSec > 0 && range ? range.startSec : 0,
				validToSec:
					endedEarly || !range ? Number.POSITIVE_INFINITY : range.endSec,
			};
		} catch (err) {
			if (
				err instanceof Error &&
				err.message.includes("Cannot open local media file")
			) {
				throw err;
			}
			// Codecs the browser's WebCodecs can't decode: its own whole-file decoder.
			if (isBrowser) return decodeWithAudioContext(url);
			if (typeof process !== "undefined" && process.versions?.node) {
				try {
					const { execFileSync } = await import("node:child_process");
					// ffmpeg's input -ss decodes from before the seek point and discards
					// up to it, so the range starts on an exact 48 kHz sample.
					const ffStartIndex = range
						? Math.round(Math.max(0, range.startSec) * 48000)
						: 0;
					const ffStartSec = ffStartIndex / 48000;
					const stdout = execFileSync(
						"ffmpeg",
						[
							"-v",
							"error",
							...(ffStartSec > 0 ? ["-ss", String(ffStartSec)] : []),
							"-i",
							url,
							...(range ? ["-t", String(range.endSec - ffStartSec)] : []),
							"-f",
							"f32le",
							"-ar",
							"48000",
							"-ac",
							"2",
							"pipe:1",
						],
						// 48 kHz stereo f32 is 384 KB/s: a fixed cap drops long sources.
						{ maxBuffer: Number.POSITIVE_INFINITY },
					);
					const totalSamples = stdout.byteLength / 4;
					const numFrames = totalSamples / 2;
					const interleaved = new Float32Array(
						stdout.buffer,
						stdout.byteOffset,
						totalSamples,
					);
					const left = new Float32Array(numFrames);
					const right = new Float32Array(numFrames);
					for (let i = 0; i < numFrames; i++) {
						left[i] = interleaved[i * 2];
						right[i] = interleaved[i * 2 + 1];
					}
					const ffEndedEarly =
						!range || ffStartSec + numFrames / 48000 < range.endSec;
					return {
						channels: [left, right],
						sampleRate: 48000,
						startIndex: ffStartIndex,
						validFromSec: ffStartSec,
						validToSec:
							ffEndedEarly || !range ? Number.POSITIVE_INFINITY : range.endSec,
					};
				} catch (ffmpegErr) {
					mediaLogger.warn(
						{ ffmpegErr },
						`[AudioExtractor] ffmpeg fallback decode failed for ${url}`,
					);
				}
			}
			mediaLogger.error(
				{ err },
				`[AudioExtractor] Failed to decode source ${url}`,
			);
			return null;
		} finally {
			if (inputAcquired) {
				inputStore.release(url);
			}
		}
	} catch (err) {
		if (
			err instanceof AudioDecodeUnavailable ||
			(err instanceof Error &&
				err.message.includes("Cannot open local media file"))
		) {
			throw err;
		}
		mediaLogger.error(
			{ err },
			`[AudioExtractor] Failed to decode source ${url}`,
		);
		return null;
	}
}

/** Samples the mapped source at the audio clock; holds are silent. */
async function resampleMappedPCM(
	input: Float32Array,
	sourceRate: number,
	outputRate: number,
	length: number,
	inputStart: number,
	map: ResolvedMediaTimeRemap,
	startFrame: number,
	fps: number,
): Promise<Float32Array> {
	const output = new Float32Array(length);
	for (let i = 0; i < length; i++) {
		if (i > 0 && i % 100_000 === 0) await yieldToMain();
		const frame = startFrame + (i / outputRate) * fps;
		if (mediaSourceIsHold(map, frame)) continue;
		const position = mediaSourceTime(map, frame, fps) * sourceRate - inputStart;
		const index = Math.floor(position);
		if (index < 0 || index >= input.length) continue;
		const a = input[index];
		output[i] = a + ((input[index + 1] ?? a) - a) * (position - index);
	}
	return output;
}

/**
 * Resamples a PCM buffer using linear interpolation to target sample rate & speed.
 * `trimOffsetSamples` is a whole-file index; `inputStartIndex` is the
 * whole-file index of `inputChannel[0]` when it holds only a range.
 */
async function resampleAndStretchPCM(
	inputChannel: Float32Array,
	sourceSampleRate: number,
	targetSampleRate: number,
	speed: number,
	outputLength: number,
	trimOffsetSamples: number,
	inputStartIndex = 0,
	firstOutputSample = 0,
): Promise<Float32Array> {
	const output = new Float32Array(outputLength);
	const resampleRatio = sourceSampleRate / targetSampleRate;
	const factor = resampleRatio * speed;

	const YIELD_INTERVAL = 100_000;

	for (let i = 0; i < outputLength; i++) {
		if (i > 0 && i % YIELD_INTERVAL === 0) {
			await yieldToMain();
		}
		// Subtracting the integer start after the sum keeps the position exact,
		// so a range resamples bit-for-bit as the whole file would.
		const srcIndex =
			trimOffsetSamples + (firstOutputSample + i) * factor - inputStartIndex;
		const indexFloor = Math.floor(srcIndex);
		const indexCeil = Math.ceil(srcIndex);

		if (indexFloor < 0) {
			output[i] = 0;
			continue;
		}
		if (indexFloor >= inputChannel.length) {
			break; // Out of bounds of source media
		}

		const weight = srcIndex - indexFloor;
		const sampleFloor = inputChannel[indexFloor] ?? 0;
		const sampleCeil =
			indexCeil >= 0 && indexCeil < inputChannel.length
				? (inputChannel[indexCeil] ?? sampleFloor)
				: sampleFloor;

		const val = sampleFloor + weight * (sampleCeil - sampleFloor);
		output[i] = Number.isFinite(val) ? val : 0;
	}

	return output;
}

/** Seconds decoded past a clip's last source sample, for interpolation. */
const RANGE_MARGIN_SEC = 0.05;

/** Processors that only scale what is already there, so silence stays silence. */
const GAIN_ONLY_PROCESSORS = new Set(["Compositor", "CompositorLayer"]);

/** Adds `src` into `dest` starting at `destStart` (which may be negative). */
/**
 * Which of a layer's gain props (`volume`, `muted`) it keys. Those the mix
 * leaves to the layer's audio processor, which reads them per sample; the
 * rest are applied as constants. Shared so both sides agree on the split.
 */
export function keyedGain(operation: unknown): {
	volume: boolean;
	muted: boolean;
} {
	const tracks =
		(
			operation as {
				animation?: {
					tracks?: ReadonlyArray<{ prop?: string; keyframes?: unknown[] }>;
				};
			} | null
		)?.animation?.tracks ?? [];
	const keys = (prop: string) =>
		tracks.some((t) => t.prop === prop && (t.keyframes?.length ?? 0) > 0);
	return { volume: keys("volume"), muted: keys("muted") };
}

function addInto(dest: Float32Array, src: Float32Array, destStart: number) {
	const from = Math.max(0, -destStart);
	const to = Math.min(src.length, dest.length - destStart);
	for (let i = from; i < to; i++) {
		dest[destStart + i] += src[i];
	}
}

/**
 * A node's stereo mix, allocated on first write. A child that covers the
 * whole bus while it is still silent is taken over instead of copied (adding
 * into silence changes nothing), so a chain of single-child nodes passes one
 * buffer up instead of allocating and summing a full-length copy per level.
 */
class MixBus {
	channels: Float32Array[] | null = null;

	constructor(readonly length: number) {}

	/** The bus channels, allocated (silent) if nothing was written yet. */
	get(): Float32Array[] {
		this.channels ??= [
			new Float32Array(this.length),
			new Float32Array(this.length),
		];
		return this.channels;
	}

	/** Whether `src` written at `start` could replace the bus outright. */
	canAdopt(src: Float32Array[], start: number): boolean {
		return (
			this.channels === null &&
			start === 0 &&
			src.every((ch) => ch.length === this.length)
		);
	}

	/** Mixes `src` into the bus starting at sample `start`. */
	add(src: Float32Array[], start: number): void {
		if (this.canAdopt(src, start) && src[0] !== src[1]) {
			this.channels = src;
			return;
		}
		const dest = this.get();
		for (let c = 0; c < 2; c++) {
			addInto(dest[c], src[c], start);
		}
	}
}

/** Progress reported by `mixAudioTracks` as each audio or video clip is mixed. */
export interface AudioMixProgress {
	/** Clips mixed so far. */
	done: number;
	/** Audio and video clips in the tree; muted or empty ones may be skipped. */
	total: number;
	/** File name of the clip just mixed (no directory or query string). */
	source?: string;
}

export interface MixAudioOptions {
	onProgress?: (progress: AudioMixProgress) => void;
	/**
	 * Mixes only [startSec, endSec) of the timeline, reading and processing
	 * only what plays there: the same samples as that stretch of a full mix
	 * (effects that carry state need lead-in before `startSec`; the caller
	 * mixes from earlier and drops it).
	 */
	window?: { startSec: number; endSec: number };
	/**
	 * Continue the effect state the last mix under this render id left,
	 * instead of starting from silence. A caller mixing back-to-back windows
	 * ([a, b) then [b, c)) gets exactly the samples of a full mix, effects
	 * with memory included. Default false.
	 */
	continue?: boolean;
	/**
	 * Apply the master stage (whole-mix auto gain, soft limiting). Default
	 * true. A windowed caller that masters once it knows the film's peak
	 * passes false and reads `peak`.
	 */
	master?: boolean;
}

export interface AudioMix {
	channels: Float32Array[];
	sampleRate: number;
	/** The loudest sample before the master stage. */
	peak: number;
}

/**
 * The master stage, in place: `gain` (the automatic ceiling: a mix whose
 * summed tracks exceed -0.5 dB, 0.95, is scaled down smoothly), then soft
 * limiting and non-finite sanitation (transparent saturation instead of
 * harsh square-wave clipping).
 */
export function applyMaster(channels: Float32Array[], gain: number): void {
	for (const channel of channels) {
		for (let i = 0; i < channel.length; i++) {
			const v = (channel[i] ?? 0) * gain;
			channel[i] = Number.isFinite(v) ? Math.tanh(v) : 0;
		}
	}
}

/** The master stage's gain for a mix whose loudest sample is `peak`. */
export function masterGain(peak: number): number {
	return peak > 0.95 ? 0.92 / peak : 1.0;
}

/** A clip's file name for progress, without paths or signed-URL queries. */
function sourceLabel(url: string): string {
	if (url.startsWith("data:")) return "inline audio";
	const name = url.split(/[?#]/)[0].split(/[\\/]/).pop();
	return name ? decodeURIComponentSafe(name) : "audio";
}

function decodeURIComponentSafe(s: string): string {
	try {
		return decodeURIComponent(s);
	} catch {
		return s;
	}
}

/**
 * Recursively parses the VirtualMediaData tree and mixes all active audio tracks
 * into a single unified stereo PCM stream (Float32Array channels).
 */
export async function mixAudioTracks(
	virtualMedia: VirtualMediaData,
	fps: number,
	targetSampleRate = 48000,
	device?: GPUDevice,
	renderId?: string,
	options: MixAudioOptions = {},
): Promise<AudioMix> {
	mediaLogger.debug(
		`[AudioExtractor] mixAudioTracks started. FPS: ${fps}, targetSampleRate: ${targetSampleRate}`,
	);
	await waitForDelayRender();

	// Effects keep state on the GPU (delay lines, reverb tails) that carries
	// across the chunks of one mix. A mix starts from silence, also when a
	// caller mixes again under the same render id (the preview, block by block).
	if (device && !options.continue)
		WebGPUAudioProcessor.resetState(renderId ?? "global", device);

	// Register all signals from the composition tree
	shaderStore.registerSignalsFromComposition(renderId, virtualMedia);

	const durationMs = getActiveMediaMetadata(virtualMedia)?.durationMs || 1000;
	const totalDurationSec = durationMs / 1000;
	const totalTargetSamples = Math.ceil(totalDurationSec * targetSampleRate);
	// The window, in timeline samples: each node mixes only its overlap.
	const winStart = options.window
		? Math.max(0, Math.round(options.window.startSec * targetSampleRate))
		: 0;
	const winEnd = options.window
		? Math.min(
				totalTargetSamples,
				Math.round(options.window.endSec * targetSampleRate),
			)
		: Number.POSITIVE_INFINITY;
	const outputSamples = options.window
		? Math.max(0, winEnd - winStart)
		: totalTargetSamples;
	/** The part [lo, hi) of a node's local samples that the window covers. */
	const visible = (absStart: number, length: number) => ({
		lo: Math.max(0, winStart - absStart),
		hi: Math.min(length, winEnd - absStart),
	});
	mediaLogger.debug(
		`[AudioExtractor] Composition total duration: ${durationMs}ms (${totalDurationSec}s). Total mixed samples to allocate: ${totalTargetSamples}`,
	);

	// Decoded ranges per source, so clips sharing a stretch of a file decode it once
	const decodedCache = new Map<string, DecodedAudioRange[]>();
	const undecodable = new Set<string>();

	async function getDecodedRange(
		url: string,
		startSec: number,
		endSec: number,
		cache = true,
	): Promise<DecodedAudioRange | null> {
		if (undecodable.has(url)) return null;
		let ranges = decodedCache.get(url);
		const hit = cache
			? ranges?.find(
					(r) => r.validFromSec <= startSec && r.validToSec >= endSec,
				)
			: undefined;
		if (hit) return hit;
		const decoded = await decodeAudioRange(url, { startSec, endSec });
		if (!decoded) {
			undecodable.add(url);
			return null;
		}
		if (!cache) return decoded;
		if (!ranges) {
			ranges = [];
			decodedCache.set(url, ranges);
		}
		ranges.push(decoded);
		return decoded;
	}

	// Limit each remapped decode by both output and source duration; never retain
	// the temporary ranges after the output chunk has been sampled.
	async function mappedChannels(
		url: string,
		length: number,
		map: ResolvedMediaTimeRemap,
		startFrame: number,
		skip = 0,
	): Promise<Float32Array[] | null> {
		const left = new Float32Array(length),
			right = new Float32Array(length);
		for (let offset = 0; offset < length; ) {
			let count = Math.min(length - offset, targetSampleRate * 2);
			const first = startFrame + ((skip + offset) / targetSampleRate) * fps;
			let range = mediaSourceRange(
				map,
				first,
				first + ((count - 1) / targetSampleRate) * fps,
				fps,
			);
			while (range.max - range.min > 3 && count > 1) {
				count = Math.max(1, Math.floor(count / 2));
				range = mediaSourceRange(
					map,
					first,
					first + ((count - 1) / targetSampleRate) * fps,
					fps,
				);
			}
			const decoded = await getDecodedRange(
				url,
				range.min,
				range.max + RANGE_MARGIN_SEC,
				false,
			);
			if (!decoded) return null;
			const a = await resampleMappedPCM(
				decoded.channels[0],
				decoded.sampleRate,
				targetSampleRate,
				count,
				decoded.startIndex,
				map,
				first,
				fps,
			);
			const b =
				decoded.channels.length > 1
					? await resampleMappedPCM(
							decoded.channels[1],
							decoded.sampleRate,
							targetSampleRate,
							count,
							decoded.startIndex,
							map,
							first,
							fps,
						)
					: a;
			left.set(a, offset);
			right.set(b, offset);
			offset += count;
		}
		return [left, right];
	}

	// A subtree without a source or a sound-producing processor mixes to
	// silence (text, images, shapes), so it is skipped instead of being
	// allocated, processed and summed at full length.
	const audibleCache = new WeakMap<VirtualMediaData, boolean>();
	function isAudible(node: VirtualMediaData): boolean {
		const cached = audibleCache.get(node);
		if (cached !== undefined) return cached;
		const op = node.operation;
		let audible = false;
		if (op?.op === "source") {
			const mediaType = getMediaType(node);
			audible = mediaType === "Audio" || mediaType === "Video";
		} else if (
			op?.op &&
			!GAIN_ONLY_PROCESSORS.has(op.op) &&
			audioRegistry.get(op.op)
		) {
			audible = true;
		}
		if (!audible && node.children) {
			audible = node.children.some((child) => child && isAudible(child));
		}
		audibleCache.set(node, audible);
		return audible;
	}

	async function traverse(
		node: VirtualMediaData,
		inheritedSeekSec: number,
		inheritedClockFrames: number,
		inheritedVolume: number,
		inheritedSpeed: number,
		inheritedDurationFrames: number | null,
		inheritedRemap?: { map: ResolvedMediaTimeRemap; offsetFrames: number },
		absStart = 0,
	): Promise<Float32Array[] | null> {
		if (!node || !isAudible(node)) return null;
		const op = node.operation;
		if (!op) {
			const durationMs = getActiveMediaMetadata(node)?.durationMs ?? 0;
			const durationFrames = Math.round((durationMs / 1000) * fps);
			const actualDurationFrames =
				inheritedRemap && inheritedDurationFrames !== null
					? inheritedDurationFrames
					: inheritedDurationFrames !== null
						? durationFrames > 0
							? Math.min(inheritedDurationFrames, durationFrames)
							: inheritedDurationFrames
						: durationFrames;

			const localSamplesCount = Math.ceil(
				(actualDurationFrames / fps) * targetSampleRate,
			);
			if (localSamplesCount <= 0) return null;
			const { lo, hi } = visible(absStart, localSamplesCount);
			if (hi <= lo) return null;

			const bus = new MixBus(hi - lo);

			for (const child of node.children) {
				const childOp = child.operation;
				const childStartFrame =
					childOp?.startFrame ?? childOp?.timeline?.startFrame ?? 0;
				const childStartSample = Math.round(
					(childStartFrame / fps) * targetSampleRate,
				);
				const childChannels = await traverse(
					child,
					inheritedSeekSec,
					inheritedClockFrames,
					inheritedVolume,
					inheritedSpeed,
					actualDurationFrames,
					inheritedRemap,
					absStart + childStartSample,
				);
				// A child's buffer starts where the window enters it.
				if (childChannels)
					bus.add(childChannels, Math.max(childStartSample, lo) - lo);
			}
			return bus.channels;
		}

		// A layer that keys its volume or mute applies them itself, over time
		// (the CompositorLayer audio processor); only resting values it does
		// not key are baked into its sources here.
		const keyed = keyedGain(op);
		const volume =
			inheritedVolume *
			(keyed.volume ? 1 : (op.volume ?? 1)) *
			(op.opacity === 0 ? 0 : 1) *
			(!keyed.muted && (op as Record<string, unknown>).muted ? 0 : 1);
		const speedMultiplier = inheritedSpeed;

		const nodeStartFrame = op.startFrame ?? op.timeline?.startFrame ?? 0;
		const ownRemap = (op as { timeRemap?: ResolvedMediaTimeRemap }).timeRemap;
		const remap = ownRemap
			? { map: ownRemap, offsetFrames: 0 }
			: inheritedRemap
				? {
						map: inheritedRemap.map,
						offsetFrames: inheritedRemap.offsetFrames + nodeStartFrame,
					}
				: undefined;
		// Compositor layer starts are flattened to composition frames by the compiler.
		const localClockFrames =
			op.op === "CompositorLayer"
				? nodeStartFrame
				: inheritedClockFrames + nodeStartFrame;

		const segments = op.timeline?.segments || [];

		// A layer's in-point: the source second that plays at its startFrame.
		// Resolved as the compositor resolves it for the picture, so a clip's
		// sound stays on its frames.
		const layer = op as {
			op: string;
			mediaStartFrame?: number;
			trimStartFrames?: number;
			trimStartSec?: number;
		};
		const layerTrimSec =
			layer.op === "CompositorLayer"
				? (layer.mediaStartFrame ?? layer.trimStartFrames ?? null) !== null
					? (layer.mediaStartFrame ?? layer.trimStartFrames ?? 0) / fps
					: Math.round((layer.trimStartSec ?? 0) * fps) / fps
				: 0;

		const nodeDurationMs = getActiveMediaMetadata(node)?.durationMs ?? 0;
		const nodeDurationFrames = Math.round((nodeDurationMs / 1000) * fps);
		const nodeActualDurationFrames =
			inheritedRemap && !ownRemap && inheritedDurationFrames !== null
				? inheritedDurationFrames
				: inheritedDurationFrames !== null
					? nodeDurationFrames > 0
						? Math.min(inheritedDurationFrames, nodeDurationFrames)
						: inheritedDurationFrames
					: nodeDurationFrames;
		const localSamplesCount = Math.ceil(
			(nodeActualDurationFrames / fps) * targetSampleRate,
		);
		if (localSamplesCount <= 0) return null;
		const { lo, hi } = visible(absStart, localSamplesCount);
		if (hi <= lo) return null;

		const bus = new MixBus(hi - lo);

		const processSegment = async (
			seg: { startSec: number; endSec?: number } | null,
			windowStartFrameOffset: number,
		) => {
			const segSeekSec = seg ? seg.startSec : 0;
			const totalSeekSec = inheritedSeekSec + layerTrimSec + segSeekSec;
			const totalClockFrames = localClockFrames + windowStartFrameOffset;

			let segDurationFrames = 0;
			if (seg) {
				segDurationFrames = Math.max(
					1,
					Math.round(((seg.endSec ?? 0) - seg.startSec) * fps),
				);
			} else {
				segDurationFrames = nodeDurationFrames;
			}

			const actualDurationFrames =
				inheritedRemap && !ownRemap && !seg && inheritedDurationFrames !== null
					? inheritedDurationFrames
					: inheritedDurationFrames !== null
						? Math.min(inheritedDurationFrames, segDurationFrames)
						: segDurationFrames;

			const segmentStartSample = Math.round(
				(windowStartFrameOffset / fps) * targetSampleRate,
			);

			// If we are on a source clip (audio/video file)
			if (op.op === "source") {
				const mediaType = getMediaType(node);
				const params = computeRenderParams(node);
				if (
					params.sourceUrl &&
					(mediaType === "Audio" || mediaType === "Video") &&
					volume > 0
				) {
					const finalSpeed = Math.max(
						0.5,
						speedMultiplier * (Number(params.speed) || 1),
					);
					const effectiveTrimSec = Math.max(
						0,
						(Number(params.trimStartSec) || 0) + totalSeekSec,
					);

					const clipDurationSec = actualDurationFrames / fps;
					const clipSamplesCount = Math.min(
						localSamplesCount - segmentStartSample,
						Math.ceil(clipDurationSec * targetSampleRate),
					);
					// The part of the clip inside the window: `skip` samples in,
					// `count` long, written at `writeAt` in the bus.
					const clipFrom = Math.max(segmentStartSample, lo);
					const clipTo = Math.min(segmentStartSample + clipSamplesCount, hi);
					const count = clipTo - clipFrom;
					const skip = clipFrom - segmentStartSample;
					const writeAt = clipFrom - lo;

					const mapped =
						remap && count > 0
							? await mappedChannels(
									params.sourceUrl,
									count,
									remap.map,
									remap.offsetFrames + windowStartFrameOffset,
									skip,
								)
							: null;
					const decoded =
						!remap && count > 0 && segmentStartSample < localSamplesCount
							? await getDecodedRange(
									params.sourceUrl,
									effectiveTrimSec + (skip / targetSampleRate) * finalSpeed,
									effectiveTrimSec +
										((skip + count) / targetSampleRate) * finalSpeed +
										RANGE_MARGIN_SEC,
								)
							: null;
					if (mapped || decoded) {
						let stretchedLeft: Float32Array;
						let stretchedRight: Float32Array;
						if (mapped) {
							[stretchedLeft, stretchedRight] = mapped;
						} else if (decoded) {
							const trimOffsetSamples = Math.round(
								effectiveTrimSec * decoded.sampleRate,
							);
							stretchedLeft = await resampleAndStretchPCM(
								decoded.channels[0],
								decoded.sampleRate,
								targetSampleRate,
								finalSpeed,
								count,
								trimOffsetSamples,
								decoded.startIndex,
								skip,
							);
							stretchedRight =
								decoded.channels.length > 1
									? await resampleAndStretchPCM(
											decoded.channels[1],
											decoded.sampleRate,
											targetSampleRate,
											finalSpeed,
											count,
											trimOffsetSamples,
											decoded.startIndex,
											skip,
										)
									: stretchedLeft;
						} else {
							throw new Error("No decoded audio");
						}
						const stretched = [stretchedLeft, stretchedRight];
						if (bus.canAdopt(stretched, writeAt)) {
							// The clip fills the silent bus: scale it in place and keep it
							if (volume !== 1) {
								for (let i = 0; i < count; i++) {
									stretchedLeft[i] *= volume;
								}
								if (stretchedRight !== stretchedLeft) {
									for (let i = 0; i < count; i++) {
										stretchedRight[i] *= volume;
									}
								}
							}
							bus.channels = [
								stretchedLeft,
								stretchedRight === stretchedLeft
									? stretchedLeft.slice()
									: stretchedRight,
							];
						} else {
							// Add to local mix with volume scaling
							const [localLeft, localRight] = bus.get();
							for (let i = 0; i < count; i++) {
								const outIndex = writeAt + i;
								localLeft[outIndex] += stretchedLeft[i] * volume;
								localRight[outIndex] += stretchedRight[i] * volume;
							}
						}
					}
					await yieldToMain();
				}
				if (mediaType === "Audio" || mediaType === "Video") {
					clipsDone++;
					options.onProgress?.({
						done: clipsDone,
						total: clipsTotal,
						...(params.sourceUrl && { source: sourceLabel(params.sourceUrl) }),
					});
				}
			}

			// Traverse children recursively
			if (node.children) {
				for (const child of node.children) {
					const childOp = child.operation;
					const childStartFrame =
						childOp?.startFrame ?? childOp?.timeline?.startFrame ?? 0;
					const relativeStartFrame =
						op.op === "CompositorLayer" && childOp?.op === "CompositorLayer"
							? childStartFrame - nodeStartFrame
							: childStartFrame;
					const childStart =
						segmentStartSample +
						Math.round((relativeStartFrame / fps) * targetSampleRate);
					const childChannels = await traverse(
						child,
						totalSeekSec,
						totalClockFrames,
						volume,
						speedMultiplier,
						actualDurationFrames,
						remap,
						absStart + childStart,
					);
					if (childChannels)
						bus.add(childChannels, Math.max(childStart, lo) - lo);
				}
			}

			// -- Plugin Processing Hook --
			const pluginProcessor = audioRegistry.get(op.op) as AudioProcessor | null;
			if (pluginProcessor) {
				try {
					const clipDurationSec = actualDurationFrames / fps;
					const clipSamplesCount = Math.min(
						localSamplesCount - segmentStartSample,
						Math.ceil(clipDurationSec * targetSampleRate),
					);

					// The part of the segment inside the window, `skip` samples in.
					const from = Math.max(segmentStartSample, lo);
					const to = Math.min(segmentStartSample + clipSamplesCount, hi);
					const skip = from - segmentStartSample;
					if (
						to > from &&
						segmentStartSample < localSamplesCount &&
						segmentStartSample >= 0
					) {
						const slicedChannels = bus
							.get()
							.map((ch) => ch.subarray(from - lo, to - lo));
						// The layer's own clock: its in-point moves the source, not its
						// animation, and a CompositorLayer's keyframes count from its
						// own first frame (as its picture's do), not the composition's.
						const layerClockFrames =
							op.op === "CompositorLayer" ? localClockFrames : 0;
						const skipSec = skip / targetSampleRate;
						const elapsedMs =
							((totalClockFrames - layerClockFrames) / fps +
								totalSeekSec -
								layerTrimSec +
								skipSec) *
							1000;
						const durationMs =
							getActiveMediaMetadata(node)?.durationMs ?? undefined;
						await pluginProcessor(slicedChannels, targetSampleRate, node, {
							device,
							frame: totalClockFrames + skipSec * fps,
							fps,
							renderId,
							elapsedMs,
							durationMs,
						});
					}
				} catch (pluginErr) {
					mediaLogger.error(
						{ err: pluginErr },
						`[AudioExtractor] Custom plugin for operation '${op.op}' failed`,
					);
				}
			}
		};

		if (segments.length > 0) {
			let currentStart = 0;
			for (const seg of segments) {
				const durSec = (seg.endSec ?? 0) - seg.startSec;
				const durFrames = Math.max(1, Math.round(durSec * fps));
				await processSegment(seg, currentStart);
				currentStart += durFrames;
			}
		} else {
			await processSegment(null, 0);
		}

		return bus.channels;
	}

	const normalized = normalizeTimeline(virtualMedia, fps);

	// Clips to mix, for progress: one per segment of each audio or video source.
	let clipsTotal = 0;
	let clipsDone = 0;
	if (options.onProgress) {
		const count = (node: VirtualMediaData | undefined) => {
			if (!node) return;
			const op = node.operation;
			if (op?.op === "source") {
				const mediaType = getMediaType(node);
				if (mediaType === "Audio" || mediaType === "Video") {
					clipsTotal += Math.max(1, op.timeline?.segments?.length ?? 0);
				}
			}
			for (const child of node.children ?? []) count(child);
		};
		count(normalized);
		options.onProgress({ done: 0, total: clipsTotal });
	}

	const rootChannels = await traverse(normalized, 0, 0, 1, 1, null);

	// Output mixed buffer is stereo; a root mix of the same length is used as is
	const mixedChannels =
		rootChannels?.every((ch) => ch.length === outputSamples) &&
		rootChannels[0] !== rootChannels[1]
			? rootChannels
			: [new Float32Array(outputSamples), new Float32Array(outputSamples)];
	const [outputLeft, outputRight] = mixedChannels;
	if (rootChannels && mixedChannels !== rootChannels) {
		for (let c = 0; c < 2; c++) {
			const dest = mixedChannels[c];
			const src = rootChannels[c];
			const len = Math.min(src.length, dest.length);
			for (let i = 0; i < len; i++) {
				dest[i] = src[i];
			}
		}
	}

	// Clear decoder cache
	decodedCache.clear();

	// Calculate master mix peak across all channels to prevent multi-track summation distortion
	let mixPeak = 0;
	for (let i = 0; i < outputSamples; i++) {
		const l = Math.abs(outputLeft[i] ?? 0);
		const r = Math.abs(outputRight[i] ?? 0);
		if (l > mixPeak) mixPeak = l;
		if (r > mixPeak) mixPeak = r;
	}

	if (options.master !== false) applyMaster(mixedChannels, masterGain(mixPeak));

	return {
		channels: mixedChannels,
		sampleRate: targetSampleRate,
		peak: mixPeak,
	};
}