Code-first video & image, rendered natively on WebGPU.
import {
getActiveMediaMetadata,
getMediaType,
mediaSourceIsHold,
mediaSourceRange,
mediaSourceTime,
type ResolvedMediaTimeRemap,
type VirtualMediaData,
} from "@framefields/core";
import { type AudioProcessor, audioRegistry } from "@framefields/node-sdk";
import {
inputStore,
resolveInputUrl,
shaderStore,
WebGPUAudioProcessor,
} from "@framefields/webgpu-renderers";
import { AudioSampleSink } from "mediabunny";
import { computeRenderParams } from "./apply-operations.js";
import { normalizeTimeline } from "./normalization.js";
// Progress chatter (every decode step) is debug output: shown only with LOG_LEVEL=debug or trace.
const verbose =
typeof process !== "undefined" &&
/^(debug|trace)$/i.test(process.env?.LOG_LEVEL ?? "");
const mediaLogger = {
debug: (msg: unknown, ...args: unknown[]) => {
if (!verbose) return;
if (typeof msg === "object" && msg !== null) {
console.info("[AudioExtractor]", msg, ...args);
} else {
console.info(`[AudioExtractor] ${msg}`, ...args);
}
},
warn: (msg: unknown, ...args: unknown[]) => {
if (typeof msg === "object" && msg !== null) {
console.warn("[AudioExtractor]", msg, ...args);
} else {
console.warn(`[AudioExtractor] ${msg}`, ...args);
}
},
error: (msg: unknown, ...args: unknown[]) => {
if (typeof msg === "object" && msg !== null) {
console.error("[AudioExtractor]", msg, ...args);
} else {
console.error(`[AudioExtractor] ${msg}`, ...args);
}
},
};
const yieldToMain = (): Promise<void> => {
if (typeof window === "undefined") {
return Promise.resolve();
}
const sched = (globalThis as Record<string, unknown>).scheduler as
| { yield?: () => Promise<void> }
| undefined;
return typeof sched?.yield === "function"
? sched.yield()
: new Promise<void>((resolve) => setTimeout(resolve, 0));
};
const waitForDelayRender = async (timeoutMs = 30_000): Promise<void> => {
await new Promise((resolve) => setTimeout(resolve, 100));
const start = Date.now();
return new Promise<void>((resolve) => {
const check = () => {
const browserHandles: unknown[] =
typeof window !== "undefined" &&
(window as unknown as { renderer_delayRenderHandles?: unknown[] })
.renderer_delayRenderHandles
? (
window as unknown as {
renderer_delayRenderHandles: unknown[];
}
).renderer_delayRenderHandles
: [];
const globalDelays =
typeof globalThis !== "undefined"
? (globalThis as any).__GATEWAI_DELAYS__
: null;
const hasBrowserDelays = browserHandles.length > 0;
const hasGlobalDelays =
globalDelays instanceof Set && globalDelays.size > 0;
if (!hasBrowserDelays && !hasGlobalDelays) {
resolve();
return;
}
if (Date.now() - start > timeoutMs) {
mediaLogger.warn(
`[AudioExtractor] waitForDelayRender timed out waiting for handles: ${JSON.stringify(
{
browserHandles,
globalDelays: globalDelays ? Array.from(globalDelays) : [],
},
)}`,
);
resolve();
return;
}
setTimeout(check, 50);
};
check();
});
};
export interface AudioClip {
sourceUrl: string;
destStartFrame: number;
trimStartSec: number;
durationFrames: number;
speed: number;
volume: number;
}
/**
* Decodes all audio samples from a given source URL into raw Float32 mono/stereo channel buffers.
*/
export async function decodeAudioSource(
url: string,
): Promise<{ channels: Float32Array[]; sampleRate: number } | null> {
const decoded = await decodeAudioRange(url);
return decoded
? { channels: decoded.channels, sampleRate: decoded.sampleRate }
: null;
}
/**
* Seconds decoded ahead of a requested range: a compressed stream (MP3, AAC)
* decoded from mid-file only matches a whole-file decode once the decoder has
* warmed up on the packets before it.
*/
const DECODE_PREROLL_SEC = 0.5;
/**
* A decoded stretch of a source. `channels[c][i]` is sample `startIndex + i`
* of a whole-file decode, so callers index it as they would the whole file.
* Samples in [validFromSec, validToSec) match a whole-file decode exactly.
*/
interface DecodedAudioRange {
channels: Float32Array[];
sampleRate: number;
startIndex: number;
validFromSec: number;
validToSec: number;
}
/**
* A source this runtime can't decode at all (a codec WebCodecs lacks, in a
* worker without AudioContext). Thrown, not mixed as silence, so the caller
* can mix where it can be decoded.
*/
export class AudioDecodeUnavailable extends Error {
override name = "AudioDecodeUnavailable";
}
/**
* The browser's whole-file decoder, for codecs WebCodecs can't decode
* (mediabunny reads byte ranges and decodes with WebCodecs first).
*/
async function decodeWithAudioContext(url: string): Promise<DecodedAudioRange> {
const AudioContextClass =
window.AudioContext ||
(window as Window & { webkitAudioContext?: typeof AudioContext })
.webkitAudioContext;
if (!AudioContextClass) {
throw new AudioDecodeUnavailable(
`No decoder here for ${url}: WebCodecs can't decode it and there is no AudioContext (a worker).`,
);
}
const audioCtx = new AudioContextClass();
try {
// Local paths load through the preview server, as the input store does.
const response = await fetch(resolveInputUrl(url, false));
mediaLogger.debug(
`[AudioExtractor] Fetch response status for ${url}: ${response.status} (${response.statusText})`,
);
if (!response.ok) {
throw new Error(`Failed to fetch audio source: ${response.statusText}`);
}
const arrayBuffer = await response.arrayBuffer();
mediaLogger.debug(
`[AudioExtractor] Fetched ${arrayBuffer.byteLength} bytes. Decoding audio data...`,
);
const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer);
mediaLogger.debug(
`[AudioExtractor] Decoded successfully: ${audioBuffer.numberOfChannels} channels, sampleRate: ${audioBuffer.sampleRate}, duration: ${audioBuffer.duration}s`,
);
const channels: Float32Array[] = [];
for (let c = 0; c < audioBuffer.numberOfChannels; c++) {
channels.push(new Float32Array(audioBuffer.getChannelData(c)));
}
// decodeAudioData only decodes whole files.
return {
channels,
sampleRate: audioBuffer.sampleRate,
startIndex: 0,
validFromSec: 0,
validToSec: Number.POSITIVE_INFINITY,
};
} finally {
void audioCtx.close();
mediaLogger.debug(`[AudioExtractor] AudioContext closed for ${url}`);
}
}
/**
* Decodes [range.startSec, range.endSec) of a source (the whole file without
* a range), so a short clip from a long file decodes only what it plays.
*/
async function decodeAudioRange(
url: string,
range?: { startSec: number; endSec: number },
): Promise<DecodedAudioRange | null> {
mediaLogger.debug(
`[AudioExtractor] decodeAudioSource started for url: ${url}`,
);
try {
// A page or a worker: anything but Node.
const isBrowser = !(
typeof process !== "undefined" && process.versions?.node
);
let inputAcquired = false;
try {
mediaLogger.debug(
`[AudioExtractor] Acquiring from inputStore for ${url}...`,
);
const input = await inputStore.acquire(url);
inputAcquired = true;
mediaLogger.debug(
`[AudioExtractor] inputStore acquired. Retrieving primary audio track...`,
);
const track = await input.getPrimaryAudioTrack();
if (!track) {
mediaLogger.warn(
`[AudioExtractor] No primary audio track found in input for ${url}`,
);
return null;
}
const sink = new AudioSampleSink(track);
const decodeFromSec = range
? Math.max(0, range.startSec - DECODE_PREROLL_SEC)
: 0;
const generator = sink.samples(decodeFromSec, range?.endSec);
const decodedPlanes: Float32Array[][] = [];
let totalFrames = 0;
let sampleRate = 48000;
let numChannels = 1;
let firstTimestamp: number | null = null;
mediaLogger.debug(
`[AudioExtractor] Starting mediabunny generator loop for ${url}...`,
);
for await (const sample of generator) {
if (firstTimestamp === null) firstTimestamp = sample.timestamp;
sampleRate = sample.sampleRate;
numChannels = sample.numberOfChannels;
const sampleFrames = sample.numberOfFrames;
if (decodedPlanes.length === 0) {
for (let c = 0; c < numChannels; c++) {
decodedPlanes.push([]);
}
}
for (let c = 0; c < numChannels; c++) {
let planeBuffer: Float32Array;
try {
const allocSize = sample.allocationSize({
planeIndex: c,
format: "f32-planar",
});
planeBuffer = new Float32Array(allocSize / 4);
sample.copyTo(planeBuffer, {
planeIndex: c,
format: "f32-planar",
});
} catch {
// Fallback: copy in sample's native format and normalize to float32 [-1, 1]
const allocSize = sample.allocationSize({ planeIndex: c });
if (sample.format === "s16" || sample.format === "s16-planar") {
const int16Buffer = new Int16Array(allocSize / 2);
sample.copyTo(int16Buffer, { planeIndex: c });
planeBuffer = new Float32Array(int16Buffer.length);
for (let i = 0; i < int16Buffer.length; i++) {
planeBuffer[i] = (int16Buffer[i] ?? 0) / 32768;
}
} else if (
sample.format === "s32" ||
sample.format === "s32-planar"
) {
const int32Buffer = new Int32Array(allocSize / 4);
sample.copyTo(int32Buffer, { planeIndex: c });
planeBuffer = new Float32Array(int32Buffer.length);
for (let i = 0; i < int32Buffer.length; i++) {
planeBuffer[i] = (int32Buffer[i] ?? 0) / 2147483648;
}
} else if (
sample.format === "u8" ||
sample.format === "u8-planar"
) {
const u8Buffer = new Uint8Array(allocSize);
sample.copyTo(u8Buffer, { planeIndex: c });
planeBuffer = new Float32Array(u8Buffer.length);
for (let i = 0; i < u8Buffer.length; i++) {
planeBuffer[i] = ((u8Buffer[i] ?? 128) - 128) / 128;
}
} else {
planeBuffer = new Float32Array(allocSize / 4);
sample.copyTo(planeBuffer, { planeIndex: c });
}
}
decodedPlanes[c].push(planeBuffer);
}
totalFrames += sampleFrames;
sample.close();
}
mediaLogger.debug(
`[AudioExtractor] Generator loop finished. Decoded total frames: ${totalFrames}`,
);
if (totalFrames === 0 || decodedPlanes.length === 0) {
return null;
}
const channels = decodedPlanes.map((planes) => {
const merged = new Float32Array(totalFrames);
let offset = 0;
for (const plane of planes) {
merged.set(plane, offset);
offset += plane.length;
}
return merged;
});
// A whole-file decode starts at the sample samples(0) yields first;
// a range is placed on that same index.
let startIndex = 0;
if (decodeFromSec > 0 && firstTimestamp !== null) {
const origin = await sink.getSample(0);
const originTimestamp =
origin?.timestamp ?? (await track.getFirstTimestamp());
origin?.close();
startIndex = Math.round(
(firstTimestamp - originTimestamp) * sampleRate,
);
}
const endedEarly =
!range || (startIndex + totalFrames) / sampleRate < range.endSec;
return {
channels,
sampleRate,
startIndex,
validFromSec: decodeFromSec > 0 && range ? range.startSec : 0,
validToSec:
endedEarly || !range ? Number.POSITIVE_INFINITY : range.endSec,
};
} catch (err) {
if (
err instanceof Error &&
err.message.includes("Cannot open local media file")
) {
throw err;
}
// Codecs the browser's WebCodecs can't decode: its own whole-file decoder.
if (isBrowser) return decodeWithAudioContext(url);
if (typeof process !== "undefined" && process.versions?.node) {
try {
const { execFileSync } = await import("node:child_process");
// ffmpeg's input -ss decodes from before the seek point and discards
// up to it, so the range starts on an exact 48 kHz sample.
const ffStartIndex = range
? Math.round(Math.max(0, range.startSec) * 48000)
: 0;
const ffStartSec = ffStartIndex / 48000;
const stdout = execFileSync(
"ffmpeg",
[
"-v",
"error",
...(ffStartSec > 0 ? ["-ss", String(ffStartSec)] : []),
"-i",
url,
...(range ? ["-t", String(range.endSec - ffStartSec)] : []),
"-f",
"f32le",
"-ar",
"48000",
"-ac",
"2",
"pipe:1",
],
// 48 kHz stereo f32 is 384 KB/s: a fixed cap drops long sources.
{ maxBuffer: Number.POSITIVE_INFINITY },
);
const totalSamples = stdout.byteLength / 4;
const numFrames = totalSamples / 2;
const interleaved = new Float32Array(
stdout.buffer,
stdout.byteOffset,
totalSamples,
);
const left = new Float32Array(numFrames);
const right = new Float32Array(numFrames);
for (let i = 0; i < numFrames; i++) {
left[i] = interleaved[i * 2];
right[i] = interleaved[i * 2 + 1];
}
const ffEndedEarly =
!range || ffStartSec + numFrames / 48000 < range.endSec;
return {
channels: [left, right],
sampleRate: 48000,
startIndex: ffStartIndex,
validFromSec: ffStartSec,
validToSec:
ffEndedEarly || !range ? Number.POSITIVE_INFINITY : range.endSec,
};
} catch (ffmpegErr) {
mediaLogger.warn(
{ ffmpegErr },
`[AudioExtractor] ffmpeg fallback decode failed for ${url}`,
);
}
}
mediaLogger.error(
{ err },
`[AudioExtractor] Failed to decode source ${url}`,
);
return null;
} finally {
if (inputAcquired) {
inputStore.release(url);
}
}
} catch (err) {
if (
err instanceof AudioDecodeUnavailable ||
(err instanceof Error &&
err.message.includes("Cannot open local media file"))
) {
throw err;
}
mediaLogger.error(
{ err },
`[AudioExtractor] Failed to decode source ${url}`,
);
return null;
}
}
/** Samples the mapped source at the audio clock; holds are silent. */
async function resampleMappedPCM(
input: Float32Array,
sourceRate: number,
outputRate: number,
length: number,
inputStart: number,
map: ResolvedMediaTimeRemap,
startFrame: number,
fps: number,
): Promise<Float32Array> {
const output = new Float32Array(length);
for (let i = 0; i < length; i++) {
if (i > 0 && i % 100_000 === 0) await yieldToMain();
const frame = startFrame + (i / outputRate) * fps;
if (mediaSourceIsHold(map, frame)) continue;
const position = mediaSourceTime(map, frame, fps) * sourceRate - inputStart;
const index = Math.floor(position);
if (index < 0 || index >= input.length) continue;
const a = input[index];
output[i] = a + ((input[index + 1] ?? a) - a) * (position - index);
}
return output;
}
/**
* Resamples a PCM buffer using linear interpolation to target sample rate & speed.
* `trimOffsetSamples` is a whole-file index; `inputStartIndex` is the
* whole-file index of `inputChannel[0]` when it holds only a range.
*/
async function resampleAndStretchPCM(
inputChannel: Float32Array,
sourceSampleRate: number,
targetSampleRate: number,
speed: number,
outputLength: number,
trimOffsetSamples: number,
inputStartIndex = 0,
firstOutputSample = 0,
): Promise<Float32Array> {
const output = new Float32Array(outputLength);
const resampleRatio = sourceSampleRate / targetSampleRate;
const factor = resampleRatio * speed;
const YIELD_INTERVAL = 100_000;
for (let i = 0; i < outputLength; i++) {
if (i > 0 && i % YIELD_INTERVAL === 0) {
await yieldToMain();
}
// Subtracting the integer start after the sum keeps the position exact,
// so a range resamples bit-for-bit as the whole file would.
const srcIndex =
trimOffsetSamples + (firstOutputSample + i) * factor - inputStartIndex;
const indexFloor = Math.floor(srcIndex);
const indexCeil = Math.ceil(srcIndex);
if (indexFloor < 0) {
output[i] = 0;
continue;
}
if (indexFloor >= inputChannel.length) {
break; // Out of bounds of source media
}
const weight = srcIndex - indexFloor;
const sampleFloor = inputChannel[indexFloor] ?? 0;
const sampleCeil =
indexCeil >= 0 && indexCeil < inputChannel.length
? (inputChannel[indexCeil] ?? sampleFloor)
: sampleFloor;
const val = sampleFloor + weight * (sampleCeil - sampleFloor);
output[i] = Number.isFinite(val) ? val : 0;
}
return output;
}
/** Seconds decoded past a clip's last source sample, for interpolation. */
const RANGE_MARGIN_SEC = 0.05;
/** Processors that only scale what is already there, so silence stays silence. */
const GAIN_ONLY_PROCESSORS = new Set(["Compositor", "CompositorLayer"]);
/** Adds `src` into `dest` starting at `destStart` (which may be negative). */
/**
* Which of a layer's gain props (`volume`, `muted`) it keys. Those the mix
* leaves to the layer's audio processor, which reads them per sample; the
* rest are applied as constants. Shared so both sides agree on the split.
*/
export function keyedGain(operation: unknown): {
volume: boolean;
muted: boolean;
} {
const tracks =
(
operation as {
animation?: {
tracks?: ReadonlyArray<{ prop?: string; keyframes?: unknown[] }>;
};
} | null
)?.animation?.tracks ?? [];
const keys = (prop: string) =>
tracks.some((t) => t.prop === prop && (t.keyframes?.length ?? 0) > 0);
return { volume: keys("volume"), muted: keys("muted") };
}
function addInto(dest: Float32Array, src: Float32Array, destStart: number) {
const from = Math.max(0, -destStart);
const to = Math.min(src.length, dest.length - destStart);
for (let i = from; i < to; i++) {
dest[destStart + i] += src[i];
}
}
/**
* A node's stereo mix, allocated on first write. A child that covers the
* whole bus while it is still silent is taken over instead of copied (adding
* into silence changes nothing), so a chain of single-child nodes passes one
* buffer up instead of allocating and summing a full-length copy per level.
*/
class MixBus {
channels: Float32Array[] | null = null;
constructor(readonly length: number) {}
/** The bus channels, allocated (silent) if nothing was written yet. */
get(): Float32Array[] {
this.channels ??= [
new Float32Array(this.length),
new Float32Array(this.length),
];
return this.channels;
}
/** Whether `src` written at `start` could replace the bus outright. */
canAdopt(src: Float32Array[], start: number): boolean {
return (
this.channels === null &&
start === 0 &&
src.every((ch) => ch.length === this.length)
);
}
/** Mixes `src` into the bus starting at sample `start`. */
add(src: Float32Array[], start: number): void {
if (this.canAdopt(src, start) && src[0] !== src[1]) {
this.channels = src;
return;
}
const dest = this.get();
for (let c = 0; c < 2; c++) {
addInto(dest[c], src[c], start);
}
}
}
/** Progress reported by `mixAudioTracks` as each audio or video clip is mixed. */
export interface AudioMixProgress {
/** Clips mixed so far. */
done: number;
/** Audio and video clips in the tree; muted or empty ones may be skipped. */
total: number;
/** File name of the clip just mixed (no directory or query string). */
source?: string;
}
export interface MixAudioOptions {
onProgress?: (progress: AudioMixProgress) => void;
/**
* Mixes only [startSec, endSec) of the timeline, reading and processing
* only what plays there: the same samples as that stretch of a full mix
* (effects that carry state need lead-in before `startSec`; the caller
* mixes from earlier and drops it).
*/
window?: { startSec: number; endSec: number };
/**
* Continue the effect state the last mix under this render id left,
* instead of starting from silence. A caller mixing back-to-back windows
* ([a, b) then [b, c)) gets exactly the samples of a full mix, effects
* with memory included. Default false.
*/
continue?: boolean;
/**
* Apply the master stage (whole-mix auto gain, soft limiting). Default
* true. A windowed caller that masters once it knows the film's peak
* passes false and reads `peak`.
*/
master?: boolean;
}
export interface AudioMix {
channels: Float32Array[];
sampleRate: number;
/** The loudest sample before the master stage. */
peak: number;
}
/**
* The master stage, in place: `gain` (the automatic ceiling: a mix whose
* summed tracks exceed -0.5 dB, 0.95, is scaled down smoothly), then soft
* limiting and non-finite sanitation (transparent saturation instead of
* harsh square-wave clipping).
*/
export function applyMaster(channels: Float32Array[], gain: number): void {
for (const channel of channels) {
for (let i = 0; i < channel.length; i++) {
const v = (channel[i] ?? 0) * gain;
channel[i] = Number.isFinite(v) ? Math.tanh(v) : 0;
}
}
}
/** The master stage's gain for a mix whose loudest sample is `peak`. */
export function masterGain(peak: number): number {
return peak > 0.95 ? 0.92 / peak : 1.0;
}
/** A clip's file name for progress, without paths or signed-URL queries. */
function sourceLabel(url: string): string {
if (url.startsWith("data:")) return "inline audio";
const name = url.split(/[?#]/)[0].split(/[\\/]/).pop();
return name ? decodeURIComponentSafe(name) : "audio";
}
function decodeURIComponentSafe(s: string): string {
try {
return decodeURIComponent(s);
} catch {
return s;
}
}
/**
* Recursively parses the VirtualMediaData tree and mixes all active audio tracks
* into a single unified stereo PCM stream (Float32Array channels).
*/
export async function mixAudioTracks(
virtualMedia: VirtualMediaData,
fps: number,
targetSampleRate = 48000,
device?: GPUDevice,
renderId?: string,
options: MixAudioOptions = {},
): Promise<AudioMix> {
mediaLogger.debug(
`[AudioExtractor] mixAudioTracks started. FPS: ${fps}, targetSampleRate: ${targetSampleRate}`,
);
await waitForDelayRender();
// Effects keep state on the GPU (delay lines, reverb tails) that carries
// across the chunks of one mix. A mix starts from silence, also when a
// caller mixes again under the same render id (the preview, block by block).
if (device && !options.continue)
WebGPUAudioProcessor.resetState(renderId ?? "global", device);
// Register all signals from the composition tree
shaderStore.registerSignalsFromComposition(renderId, virtualMedia);
const durationMs = getActiveMediaMetadata(virtualMedia)?.durationMs || 1000;
const totalDurationSec = durationMs / 1000;
const totalTargetSamples = Math.ceil(totalDurationSec * targetSampleRate);
// The window, in timeline samples: each node mixes only its overlap.
const winStart = options.window
? Math.max(0, Math.round(options.window.startSec * targetSampleRate))
: 0;
const winEnd = options.window
? Math.min(
totalTargetSamples,
Math.round(options.window.endSec * targetSampleRate),
)
: Number.POSITIVE_INFINITY;
const outputSamples = options.window
? Math.max(0, winEnd - winStart)
: totalTargetSamples;
/** The part [lo, hi) of a node's local samples that the window covers. */
const visible = (absStart: number, length: number) => ({
lo: Math.max(0, winStart - absStart),
hi: Math.min(length, winEnd - absStart),
});
mediaLogger.debug(
`[AudioExtractor] Composition total duration: ${durationMs}ms (${totalDurationSec}s). Total mixed samples to allocate: ${totalTargetSamples}`,
);
// Decoded ranges per source, so clips sharing a stretch of a file decode it once
const decodedCache = new Map<string, DecodedAudioRange[]>();
const undecodable = new Set<string>();
async function getDecodedRange(
url: string,
startSec: number,
endSec: number,
cache = true,
): Promise<DecodedAudioRange | null> {
if (undecodable.has(url)) return null;
let ranges = decodedCache.get(url);
const hit = cache
? ranges?.find(
(r) => r.validFromSec <= startSec && r.validToSec >= endSec,
)
: undefined;
if (hit) return hit;
const decoded = await decodeAudioRange(url, { startSec, endSec });
if (!decoded) {
undecodable.add(url);
return null;
}
if (!cache) return decoded;
if (!ranges) {
ranges = [];
decodedCache.set(url, ranges);
}
ranges.push(decoded);
return decoded;
}
// Limit each remapped decode by both output and source duration; never retain
// the temporary ranges after the output chunk has been sampled.
async function mappedChannels(
url: string,
length: number,
map: ResolvedMediaTimeRemap,
startFrame: number,
skip = 0,
): Promise<Float32Array[] | null> {
const left = new Float32Array(length),
right = new Float32Array(length);
for (let offset = 0; offset < length; ) {
let count = Math.min(length - offset, targetSampleRate * 2);
const first = startFrame + ((skip + offset) / targetSampleRate) * fps;
let range = mediaSourceRange(
map,
first,
first + ((count - 1) / targetSampleRate) * fps,
fps,
);
while (range.max - range.min > 3 && count > 1) {
count = Math.max(1, Math.floor(count / 2));
range = mediaSourceRange(
map,
first,
first + ((count - 1) / targetSampleRate) * fps,
fps,
);
}
const decoded = await getDecodedRange(
url,
range.min,
range.max + RANGE_MARGIN_SEC,
false,
);
if (!decoded) return null;
const a = await resampleMappedPCM(
decoded.channels[0],
decoded.sampleRate,
targetSampleRate,
count,
decoded.startIndex,
map,
first,
fps,
);
const b =
decoded.channels.length > 1
? await resampleMappedPCM(
decoded.channels[1],
decoded.sampleRate,
targetSampleRate,
count,
decoded.startIndex,
map,
first,
fps,
)
: a;
left.set(a, offset);
right.set(b, offset);
offset += count;
}
return [left, right];
}
// A subtree without a source or a sound-producing processor mixes to
// silence (text, images, shapes), so it is skipped instead of being
// allocated, processed and summed at full length.
const audibleCache = new WeakMap<VirtualMediaData, boolean>();
function isAudible(node: VirtualMediaData): boolean {
const cached = audibleCache.get(node);
if (cached !== undefined) return cached;
const op = node.operation;
let audible = false;
if (op?.op === "source") {
const mediaType = getMediaType(node);
audible = mediaType === "Audio" || mediaType === "Video";
} else if (
op?.op &&
!GAIN_ONLY_PROCESSORS.has(op.op) &&
audioRegistry.get(op.op)
) {
audible = true;
}
if (!audible && node.children) {
audible = node.children.some((child) => child && isAudible(child));
}
audibleCache.set(node, audible);
return audible;
}
async function traverse(
node: VirtualMediaData,
inheritedSeekSec: number,
inheritedClockFrames: number,
inheritedVolume: number,
inheritedSpeed: number,
inheritedDurationFrames: number | null,
inheritedRemap?: { map: ResolvedMediaTimeRemap; offsetFrames: number },
absStart = 0,
): Promise<Float32Array[] | null> {
if (!node || !isAudible(node)) return null;
const op = node.operation;
if (!op) {
const durationMs = getActiveMediaMetadata(node)?.durationMs ?? 0;
const durationFrames = Math.round((durationMs / 1000) * fps);
const actualDurationFrames =
inheritedRemap && inheritedDurationFrames !== null
? inheritedDurationFrames
: inheritedDurationFrames !== null
? durationFrames > 0
? Math.min(inheritedDurationFrames, durationFrames)
: inheritedDurationFrames
: durationFrames;
const localSamplesCount = Math.ceil(
(actualDurationFrames / fps) * targetSampleRate,
);
if (localSamplesCount <= 0) return null;
const { lo, hi } = visible(absStart, localSamplesCount);
if (hi <= lo) return null;
const bus = new MixBus(hi - lo);
for (const child of node.children) {
const childOp = child.operation;
const childStartFrame =
childOp?.startFrame ?? childOp?.timeline?.startFrame ?? 0;
const childStartSample = Math.round(
(childStartFrame / fps) * targetSampleRate,
);
const childChannels = await traverse(
child,
inheritedSeekSec,
inheritedClockFrames,
inheritedVolume,
inheritedSpeed,
actualDurationFrames,
inheritedRemap,
absStart + childStartSample,
);
// A child's buffer starts where the window enters it.
if (childChannels)
bus.add(childChannels, Math.max(childStartSample, lo) - lo);
}
return bus.channels;
}
// A layer that keys its volume or mute applies them itself, over time
// (the CompositorLayer audio processor); only resting values it does
// not key are baked into its sources here.
const keyed = keyedGain(op);
const volume =
inheritedVolume *
(keyed.volume ? 1 : (op.volume ?? 1)) *
(op.opacity === 0 ? 0 : 1) *
(!keyed.muted && (op as Record<string, unknown>).muted ? 0 : 1);
const speedMultiplier = inheritedSpeed;
const nodeStartFrame = op.startFrame ?? op.timeline?.startFrame ?? 0;
const ownRemap = (op as { timeRemap?: ResolvedMediaTimeRemap }).timeRemap;
const remap = ownRemap
? { map: ownRemap, offsetFrames: 0 }
: inheritedRemap
? {
map: inheritedRemap.map,
offsetFrames: inheritedRemap.offsetFrames + nodeStartFrame,
}
: undefined;
// Compositor layer starts are flattened to composition frames by the compiler.
const localClockFrames =
op.op === "CompositorLayer"
? nodeStartFrame
: inheritedClockFrames + nodeStartFrame;
const segments = op.timeline?.segments || [];
// A layer's in-point: the source second that plays at its startFrame.
// Resolved as the compositor resolves it for the picture, so a clip's
// sound stays on its frames.
const layer = op as {
op: string;
mediaStartFrame?: number;
trimStartFrames?: number;
trimStartSec?: number;
};
const layerTrimSec =
layer.op === "CompositorLayer"
? (layer.mediaStartFrame ?? layer.trimStartFrames ?? null) !== null
? (layer.mediaStartFrame ?? layer.trimStartFrames ?? 0) / fps
: Math.round((layer.trimStartSec ?? 0) * fps) / fps
: 0;
const nodeDurationMs = getActiveMediaMetadata(node)?.durationMs ?? 0;
const nodeDurationFrames = Math.round((nodeDurationMs / 1000) * fps);
const nodeActualDurationFrames =
inheritedRemap && !ownRemap && inheritedDurationFrames !== null
? inheritedDurationFrames
: inheritedDurationFrames !== null
? nodeDurationFrames > 0
? Math.min(inheritedDurationFrames, nodeDurationFrames)
: inheritedDurationFrames
: nodeDurationFrames;
const localSamplesCount = Math.ceil(
(nodeActualDurationFrames / fps) * targetSampleRate,
);
if (localSamplesCount <= 0) return null;
const { lo, hi } = visible(absStart, localSamplesCount);
if (hi <= lo) return null;
const bus = new MixBus(hi - lo);
const processSegment = async (
seg: { startSec: number; endSec?: number } | null,
windowStartFrameOffset: number,
) => {
const segSeekSec = seg ? seg.startSec : 0;
const totalSeekSec = inheritedSeekSec + layerTrimSec + segSeekSec;
const totalClockFrames = localClockFrames + windowStartFrameOffset;
let segDurationFrames = 0;
if (seg) {
segDurationFrames = Math.max(
1,
Math.round(((seg.endSec ?? 0) - seg.startSec) * fps),
);
} else {
segDurationFrames = nodeDurationFrames;
}
const actualDurationFrames =
inheritedRemap && !ownRemap && !seg && inheritedDurationFrames !== null
? inheritedDurationFrames
: inheritedDurationFrames !== null
? Math.min(inheritedDurationFrames, segDurationFrames)
: segDurationFrames;
const segmentStartSample = Math.round(
(windowStartFrameOffset / fps) * targetSampleRate,
);
// If we are on a source clip (audio/video file)
if (op.op === "source") {
const mediaType = getMediaType(node);
const params = computeRenderParams(node);
if (
params.sourceUrl &&
(mediaType === "Audio" || mediaType === "Video") &&
volume > 0
) {
const finalSpeed = Math.max(
0.5,
speedMultiplier * (Number(params.speed) || 1),
);
const effectiveTrimSec = Math.max(
0,
(Number(params.trimStartSec) || 0) + totalSeekSec,
);
const clipDurationSec = actualDurationFrames / fps;
const clipSamplesCount = Math.min(
localSamplesCount - segmentStartSample,
Math.ceil(clipDurationSec * targetSampleRate),
);
// The part of the clip inside the window: `skip` samples in,
// `count` long, written at `writeAt` in the bus.
const clipFrom = Math.max(segmentStartSample, lo);
const clipTo = Math.min(segmentStartSample + clipSamplesCount, hi);
const count = clipTo - clipFrom;
const skip = clipFrom - segmentStartSample;
const writeAt = clipFrom - lo;
const mapped =
remap && count > 0
? await mappedChannels(
params.sourceUrl,
count,
remap.map,
remap.offsetFrames + windowStartFrameOffset,
skip,
)
: null;
const decoded =
!remap && count > 0 && segmentStartSample < localSamplesCount
? await getDecodedRange(
params.sourceUrl,
effectiveTrimSec + (skip / targetSampleRate) * finalSpeed,
effectiveTrimSec +
((skip + count) / targetSampleRate) * finalSpeed +
RANGE_MARGIN_SEC,
)
: null;
if (mapped || decoded) {
let stretchedLeft: Float32Array;
let stretchedRight: Float32Array;
if (mapped) {
[stretchedLeft, stretchedRight] = mapped;
} else if (decoded) {
const trimOffsetSamples = Math.round(
effectiveTrimSec * decoded.sampleRate,
);
stretchedLeft = await resampleAndStretchPCM(
decoded.channels[0],
decoded.sampleRate,
targetSampleRate,
finalSpeed,
count,
trimOffsetSamples,
decoded.startIndex,
skip,
);
stretchedRight =
decoded.channels.length > 1
? await resampleAndStretchPCM(
decoded.channels[1],
decoded.sampleRate,
targetSampleRate,
finalSpeed,
count,
trimOffsetSamples,
decoded.startIndex,
skip,
)
: stretchedLeft;
} else {
throw new Error("No decoded audio");
}
const stretched = [stretchedLeft, stretchedRight];
if (bus.canAdopt(stretched, writeAt)) {
// The clip fills the silent bus: scale it in place and keep it
if (volume !== 1) {
for (let i = 0; i < count; i++) {
stretchedLeft[i] *= volume;
}
if (stretchedRight !== stretchedLeft) {
for (let i = 0; i < count; i++) {
stretchedRight[i] *= volume;
}
}
}
bus.channels = [
stretchedLeft,
stretchedRight === stretchedLeft
? stretchedLeft.slice()
: stretchedRight,
];
} else {
// Add to local mix with volume scaling
const [localLeft, localRight] = bus.get();
for (let i = 0; i < count; i++) {
const outIndex = writeAt + i;
localLeft[outIndex] += stretchedLeft[i] * volume;
localRight[outIndex] += stretchedRight[i] * volume;
}
}
}
await yieldToMain();
}
if (mediaType === "Audio" || mediaType === "Video") {
clipsDone++;
options.onProgress?.({
done: clipsDone,
total: clipsTotal,
...(params.sourceUrl && { source: sourceLabel(params.sourceUrl) }),
});
}
}
// Traverse children recursively
if (node.children) {
for (const child of node.children) {
const childOp = child.operation;
const childStartFrame =
childOp?.startFrame ?? childOp?.timeline?.startFrame ?? 0;
const relativeStartFrame =
op.op === "CompositorLayer" && childOp?.op === "CompositorLayer"
? childStartFrame - nodeStartFrame
: childStartFrame;
const childStart =
segmentStartSample +
Math.round((relativeStartFrame / fps) * targetSampleRate);
const childChannels = await traverse(
child,
totalSeekSec,
totalClockFrames,
volume,
speedMultiplier,
actualDurationFrames,
remap,
absStart + childStart,
);
if (childChannels)
bus.add(childChannels, Math.max(childStart, lo) - lo);
}
}
// -- Plugin Processing Hook --
const pluginProcessor = audioRegistry.get(op.op) as AudioProcessor | null;
if (pluginProcessor) {
try {
const clipDurationSec = actualDurationFrames / fps;
const clipSamplesCount = Math.min(
localSamplesCount - segmentStartSample,
Math.ceil(clipDurationSec * targetSampleRate),
);
// The part of the segment inside the window, `skip` samples in.
const from = Math.max(segmentStartSample, lo);
const to = Math.min(segmentStartSample + clipSamplesCount, hi);
const skip = from - segmentStartSample;
if (
to > from &&
segmentStartSample < localSamplesCount &&
segmentStartSample >= 0
) {
const slicedChannels = bus
.get()
.map((ch) => ch.subarray(from - lo, to - lo));
// The layer's own clock: its in-point moves the source, not its
// animation, and a CompositorLayer's keyframes count from its
// own first frame (as its picture's do), not the composition's.
const layerClockFrames =
op.op === "CompositorLayer" ? localClockFrames : 0;
const skipSec = skip / targetSampleRate;
const elapsedMs =
((totalClockFrames - layerClockFrames) / fps +
totalSeekSec -
layerTrimSec +
skipSec) *
1000;
const durationMs =
getActiveMediaMetadata(node)?.durationMs ?? undefined;
await pluginProcessor(slicedChannels, targetSampleRate, node, {
device,
frame: totalClockFrames + skipSec * fps,
fps,
renderId,
elapsedMs,
durationMs,
});
}
} catch (pluginErr) {
mediaLogger.error(
{ err: pluginErr },
`[AudioExtractor] Custom plugin for operation '${op.op}' failed`,
);
}
}
};
if (segments.length > 0) {
let currentStart = 0;
for (const seg of segments) {
const durSec = (seg.endSec ?? 0) - seg.startSec;
const durFrames = Math.max(1, Math.round(durSec * fps));
await processSegment(seg, currentStart);
currentStart += durFrames;
}
} else {
await processSegment(null, 0);
}
return bus.channels;
}
const normalized = normalizeTimeline(virtualMedia, fps);
// Clips to mix, for progress: one per segment of each audio or video source.
let clipsTotal = 0;
let clipsDone = 0;
if (options.onProgress) {
const count = (node: VirtualMediaData | undefined) => {
if (!node) return;
const op = node.operation;
if (op?.op === "source") {
const mediaType = getMediaType(node);
if (mediaType === "Audio" || mediaType === "Video") {
clipsTotal += Math.max(1, op.timeline?.segments?.length ?? 0);
}
}
for (const child of node.children ?? []) count(child);
};
count(normalized);
options.onProgress({ done: 0, total: clipsTotal });
}
const rootChannels = await traverse(normalized, 0, 0, 1, 1, null);
// Output mixed buffer is stereo; a root mix of the same length is used as is
const mixedChannels =
rootChannels?.every((ch) => ch.length === outputSamples) &&
rootChannels[0] !== rootChannels[1]
? rootChannels
: [new Float32Array(outputSamples), new Float32Array(outputSamples)];
const [outputLeft, outputRight] = mixedChannels;
if (rootChannels && mixedChannels !== rootChannels) {
for (let c = 0; c < 2; c++) {
const dest = mixedChannels[c];
const src = rootChannels[c];
const len = Math.min(src.length, dest.length);
for (let i = 0; i < len; i++) {
dest[i] = src[i];
}
}
}
// Clear decoder cache
decodedCache.clear();
// Calculate master mix peak across all channels to prevent multi-track summation distortion
let mixPeak = 0;
for (let i = 0; i < outputSamples; i++) {
const l = Math.abs(outputLeft[i] ?? 0);
const r = Math.abs(outputRight[i] ?? 0);
if (l > mixPeak) mixPeak = l;
if (r > mixPeak) mixPeak = r;
}
if (options.master !== false) applyMaster(mixedChannels, masterGain(mixPeak));
return {
channels: mixedChannels,
sampleRate: targetSampleRate,
peak: mixPeak,
};
}