Skip to content

Code-first video & image, rendered natively on WebGPU.

Speed up audio mixing for long timelines

Claudefa5c2fd1624d
diff --git a/packages/compositions/src/utils/audio-extractor.test.ts b/packages/compositions/src/utils/audio-extractor.test.ts
index 7fea9406..8753c402 100644
--- a/packages/compositions/src/utils/audio-extractor.test.ts
+++ b/packages/compositions/src/utils/audio-extractor.test.ts
@@ -1,8 +1,19 @@
+import { execFileSync } from "node:child_process";
import fs from "node:fs/promises";
+import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { decodeAudioSource } from "./audio-extractor.js";
+const hasFfmpeg = (() => {
+ try {
+ execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });
+ return true;
+ } catch {
+ return false;
+ }
+})();
+
function encodeTestWav(
samples: Float32Array,
sampleRate = 48000,
@@ -72,4 +83,28 @@ describe("Audio Extractor: decodeAudioSource", () => {
expect(maxAbs).toBeGreaterThan(0.7);
expect(maxAbs).toBeLessThanOrEqual(0.76);
});
+
+ it.skipIf(!hasFfmpeg)(
+ "decodes a source longer than a few minutes in full",
+ async () => {
+ // 48 kHz stereo float is 384 KB/s: five minutes is past any small
+ // output cap a decoder pipe might have.
+ const file = path.join(
+ await fs.mkdtemp(path.join(os.tmpdir(), "audio-extractor-")),
+ "long.mp3",
+ );
+ execFileSync("ffmpeg", [
+ ...["-v", "error", "-y", "-f", "lavfi"],
+ ...["-i", "sine=frequency=440:sample_rate=48000:duration=300"],
+ ...["-ac", "2", "-c:a", "libmp3lame", "-b:a", "64k", file],
+ ]);
+
+ const result = await decodeAudioSource(file);
+ expect(result?.channels.length).toBe(2);
+ expect(result?.channels[0]?.length ?? 0).toBeGreaterThanOrEqual(
+ 299 * (result?.sampleRate ?? 0),
+ );
+ },
+ 120_000,
+ );
});
diff --git a/packages/compositions/src/utils/audio-extractor.ts b/packages/compositions/src/utils/audio-extractor.ts
index c604e8e6..fe14e0ad 100644
--- a/packages/compositions/src/utils/audio-extractor.ts
+++ b/packages/compositions/src/utils/audio-extractor.ts
@@ -116,6 +116,40 @@ export interface AudioClip {
export async function decodeAudioSource(
url: string,
): Promise<{ channels: Float32Array[]; sampleRate: number } | null> {
+ const decoded = await decodeAudioRange(url);
+ return decoded
+ ? { channels: decoded.channels, sampleRate: decoded.sampleRate }
+ : null;
+}
+
+/**
+ * Seconds decoded ahead of a requested range: a compressed stream (MP3, AAC)
+ * decoded from mid-file only matches a whole-file decode once the decoder has
+ * warmed up on the packets before it.
+ */
+const DECODE_PREROLL_SEC = 0.5;
+
+/**
+ * A decoded stretch of a source. `channels[c][i]` is sample `startIndex + i`
+ * of a whole-file decode, so callers index it as they would the whole file.
+ * Samples in [validFromSec, validToSec) match a whole-file decode exactly.
+ */
+interface DecodedAudioRange {
+ channels: Float32Array[];
+ sampleRate: number;
+ startIndex: number;
+ validFromSec: number;
+ validToSec: number;
+}
+
+/**
+ * Decodes [range.startSec, range.endSec) of a source (the whole file without
+ * a range), so a short clip from a long file decodes only what it plays.
+ */
+async function decodeAudioRange(
+ url: string,
+ range?: { startSec: number; endSec: number },
+): Promise<DecodedAudioRange | null> {
mediaLogger.debug(
`[AudioExtractor] decodeAudioSource started for url: ${url}`,
);
@@ -161,7 +195,14 @@ export async function decodeAudioSource(
for (let c = 0; c < audioBuffer.numberOfChannels; c++) {
channels.push(new Float32Array(audioBuffer.getChannelData(c)));
}
- return { channels, sampleRate: audioBuffer.sampleRate };
+ // decodeAudioData only decodes whole files.
+ return {
+ channels,
+ sampleRate: audioBuffer.sampleRate,
+ startIndex: 0,
+ validFromSec: 0,
+ validToSec: Number.POSITIVE_INFINITY,
+ };
} finally {
void audioCtx.close();
mediaLogger.debug(`[AudioExtractor] AudioContext closed for ${url}`);
@@ -187,17 +228,22 @@ export async function decodeAudioSource(
}
const sink = new AudioSampleSink(track);
- const generator = sink.samples(0);
+ const decodeFromSec = range
+ ? Math.max(0, range.startSec - DECODE_PREROLL_SEC)
+ : 0;
+ const generator = sink.samples(decodeFromSec, range?.endSec);
const decodedPlanes: Float32Array[][] = [];
let totalFrames = 0;
let sampleRate = 48000;
let numChannels = 1;
+ let firstTimestamp: number | null = null;
mediaLogger.debug(
`[AudioExtractor] Starting mediabunny generator loop for ${url}...`,
);
for await (const sample of generator) {
+ if (firstTimestamp === null) firstTimestamp = sample.timestamp;
sampleRate = sample.sampleRate;
numChannels = sample.numberOfChannels;
const sampleFrames = sample.numberOfFrames;
@@ -279,7 +325,29 @@ export async function decodeAudioSource(
return merged;
});
- return { channels, sampleRate };
+ // A whole-file decode starts at the sample samples(0) yields first;
+ // a range is placed on that same index.
+ let startIndex = 0;
+ if (decodeFromSec > 0 && firstTimestamp !== null) {
+ const origin = await sink.getSample(0);
+ const originTimestamp =
+ origin?.timestamp ?? (await track.getFirstTimestamp());
+ origin?.close();
+ startIndex = Math.round(
+ (firstTimestamp - originTimestamp) * sampleRate,
+ );
+ }
+ const endedEarly =
+ !range || (startIndex + totalFrames) / sampleRate < range.endSec;
+
+ return {
+ channels,
+ sampleRate,
+ startIndex,
+ validFromSec: decodeFromSec > 0 && range ? range.startSec : 0,
+ validToSec:
+ endedEarly || !range ? Number.POSITIVE_INFINITY : range.endSec,
+ };
} catch (err) {
if (
err instanceof Error &&
@@ -289,10 +357,32 @@ export async function decodeAudioSource(
}
if (typeof process !== "undefined" && process.versions?.node) {
try {
- const { execSync } = await import("node:child_process");
- const stdout = execSync(
- `ffmpeg -v error -i "${url}" -f f32le -ar 48000 -ac 2 pipe:1`,
- { maxBuffer: 100 * 1024 * 1024 },
+ const { execFileSync } = await import("node:child_process");
+ // ffmpeg's input -ss decodes from before the seek point and discards
+ // up to it, so the range starts on an exact 48 kHz sample.
+ const ffStartIndex = range
+ ? Math.round(Math.max(0, range.startSec) * 48000)
+ : 0;
+ const ffStartSec = ffStartIndex / 48000;
+ const stdout = execFileSync(
+ "ffmpeg",
+ [
+ "-v",
+ "error",
+ ...(ffStartSec > 0 ? ["-ss", String(ffStartSec)] : []),
+ "-i",
+ url,
+ ...(range ? ["-t", String(range.endSec - ffStartSec)] : []),
+ "-f",
+ "f32le",
+ "-ar",
+ "48000",
+ "-ac",
+ "2",
+ "pipe:1",
+ ],
+ // 48 kHz stereo f32 is 384 KB/s: a fixed cap drops long sources.
+ { maxBuffer: Number.POSITIVE_INFINITY },
);
const totalSamples = stdout.byteLength / 4;
const numFrames = totalSamples / 2;
@@ -307,7 +397,16 @@ export async function decodeAudioSource(
left[i] = interleaved[i * 2];
right[i] = interleaved[i * 2 + 1];
}
- return { channels: [left, right], sampleRate: 48000 };
+ const ffEndedEarly =
+ !range || ffStartSec + numFrames / 48000 < range.endSec;
+ return {
+ channels: [left, right],
+ sampleRate: 48000,
+ startIndex: ffStartIndex,
+ validFromSec: ffStartSec,
+ validToSec:
+ ffEndedEarly || !range ? Number.POSITIVE_INFINITY : range.endSec,
+ };
} catch (ffmpegErr) {
mediaLogger.warn(
{ ffmpegErr },
@@ -342,6 +441,8 @@ export async function decodeAudioSource(
/**
* Resamples a PCM buffer using linear interpolation to target sample rate & speed.
+ * `trimOffsetSamples` is a whole-file index; `inputStartIndex` is the
+ * whole-file index of `inputChannel[0]` when it holds only a range.
*/
async function resampleAndStretchPCM(
inputChannel: Float32Array,
@@ -350,6 +451,7 @@ async function resampleAndStretchPCM(
speed: number,
outputLength: number,
trimOffsetSamples: number,
+ inputStartIndex = 0,
): Promise<Float32Array> {
const output = new Float32Array(outputLength);
const resampleRatio = sourceSampleRate / targetSampleRate;
@@ -361,7 +463,9 @@ async function resampleAndStretchPCM(
if (i > 0 && i % YIELD_INTERVAL === 0) {
await yieldToMain();
}
- const srcIndex = trimOffsetSamples + i * factor;
+ // Subtracting the integer start after the sum keeps the position exact,
+ // so a range resamples bit-for-bit as the whole file would.
+ const srcIndex = trimOffsetSamples + i * factor - inputStartIndex;
const indexFloor = Math.floor(srcIndex);
const indexCeil = Math.ceil(srcIndex);
@@ -387,6 +491,63 @@ async function resampleAndStretchPCM(
return output;
}
+/** Seconds decoded past a clip's last source sample, for interpolation. */
+const RANGE_MARGIN_SEC = 0.05;
+
+/** Processors that only scale what is already there, so silence stays silence. */
+const GAIN_ONLY_PROCESSORS = new Set(["Compositor", "CompositorLayer"]);
+
+/** Adds `src` into `dest` starting at `destStart` (which may be negative). */
+function addInto(dest: Float32Array, src: Float32Array, destStart: number) {
+ const from = Math.max(0, -destStart);
+ const to = Math.min(src.length, dest.length - destStart);
+ for (let i = from; i < to; i++) {
+ dest[destStart + i] += src[i];
+ }
+}
+
+/**
+ * A node's stereo mix, allocated on first write. A child that covers the
+ * whole bus while it is still silent is taken over instead of copied (adding
+ * into silence changes nothing), so a chain of single-child nodes passes one
+ * buffer up instead of allocating and summing a full-length copy per level.
+ */
+class MixBus {
+ channels: Float32Array[] | null = null;
+
+ constructor(readonly length: number) {}
+
+ /** The bus channels, allocated (silent) if nothing was written yet. */
+ get(): Float32Array[] {
+ this.channels ??= [
+ new Float32Array(this.length),
+ new Float32Array(this.length),
+ ];
+ return this.channels;
+ }
+
+ /** Whether `src` written at `start` could replace the bus outright. */
+ canAdopt(src: Float32Array[], start: number): boolean {
+ return (
+ this.channels === null &&
+ start === 0 &&
+ src.every((ch) => ch.length === this.length)
+ );
+ }
+
+ /** Mixes `src` into the bus starting at sample `start`. */
+ add(src: Float32Array[], start: number): void {
+ if (this.canAdopt(src, start) && src[0] !== src[1]) {
+ this.channels = src;
+ return;
+ }
+ const dest = this.get();
+ for (let c = 0; c < 2; c++) {
+ addInto(dest[c], src[c], start);
+ }
+ }
+}
+
/**
* Recursively parses the VirtualMediaData tree and mixes all active audio tracks
* into a single unified stereo PCM stream (Float32Array channels).
@@ -413,16 +574,59 @@ export async function mixAudioTracks(
`[AudioExtractor] Composition total duration: ${durationMs}ms (${totalDurationSec}s). Total mixed samples to allocate: ${totalTargetSamples}`,
);
- // Output mixed buffer is stereo
- const outputLeft = new Float32Array(totalTargetSamples);
- const outputRight = new Float32Array(totalTargetSamples);
- const mixedChannels = [outputLeft, outputRight];
+ // Decoded ranges per source, so clips sharing a stretch of a file decode it once
+ const decodedCache = new Map<string, DecodedAudioRange[]>();
+ const undecodable = new Set<string>();
+
+ async function getDecodedRange(
+ url: string,
+ startSec: number,
+ endSec: number,
+ ): Promise<DecodedAudioRange | null> {
+ if (undecodable.has(url)) return null;
+ let ranges = decodedCache.get(url);
+ const hit = ranges?.find(
+ (r) => r.validFromSec <= startSec && r.validToSec >= endSec,
+ );
+ if (hit) return hit;
+ const decoded = await decodeAudioRange(url, { startSec, endSec });
+ if (!decoded) {
+ undecodable.add(url);
+ return null;
+ }
+ if (!ranges) {
+ ranges = [];
+ decodedCache.set(url, ranges);
+ }
+ ranges.push(decoded);
+ return decoded;
+ }
- // Map to cache decoded sources so we don't decode multiple clips of the same file twice
- const decodedCache = new Map<
- string,
- { channels: Float32Array[]; sampleRate: number } | null
- >();
+ // A subtree without a source or a sound-producing processor mixes to
+ // silence (text, images, shapes), so it is skipped instead of being
+ // allocated, processed and summed at full length.
+ const audibleCache = new WeakMap<VirtualMediaData, boolean>();
+ function isAudible(node: VirtualMediaData): boolean {
+ const cached = audibleCache.get(node);
+ if (cached !== undefined) return cached;
+ const op = node.operation;
+ let audible = false;
+ if (op?.op === "source") {
+ const mediaType = getMediaType(node);
+ audible = mediaType === "Audio" || mediaType === "Video";
+ } else if (
+ op?.op &&
+ !GAIN_ONLY_PROCESSORS.has(op.op) &&
+ audioRegistry.get(op.op)
+ ) {
+ audible = true;
+ }
+ if (!audible && node.children) {
+ audible = node.children.some((child) => child && isAudible(child));
+ }
+ audibleCache.set(node, audible);
+ return audible;
+ }
async function traverse(
node: VirtualMediaData,
@@ -432,7 +636,7 @@ export async function mixAudioTracks(
inheritedSpeed: number,
inheritedDurationFrames: number | null,
): Promise<Float32Array[] | null> {
- if (!node) return null;
+ if (!node || !isAudible(node)) return null;
const op = node.operation;
if (!op) {
const durationMs = getActiveMediaMetadata(node)?.durationMs ?? 0;
@@ -449,9 +653,7 @@ export async function mixAudioTracks(
);
if (localSamplesCount <= 0) return null;
- const localLeft = new Float32Array(localSamplesCount);
- const localRight = new Float32Array(localSamplesCount);
- const localChannels = [localLeft, localRight];
+ const bus = new MixBus(localSamplesCount);
for (const child of node.children) {
const childChannels = await traverse(
@@ -469,19 +671,10 @@ export async function mixAudioTracks(
const childStartSample = Math.round(
(childStartFrame / fps) * targetSampleRate,
);
- for (let c = 0; c < 2; c++) {
- const dest = localChannels[c];
- const src = childChannels[c];
- const len = Math.min(src.length, dest.length - childStartSample);
- for (let i = 0; i < len; i++) {
- if (childStartSample + i >= 0) {
- dest[childStartSample + i] += src[i];
- }
- }
- }
+ bus.add(childChannels, childStartSample);
}
}
- return localChannels;
+ return bus.channels;
}
const volume =
@@ -525,9 +718,7 @@ export async function mixAudioTracks(
);
if (localSamplesCount <= 0) return null;
- const localLeft = new Float32Array(localSamplesCount);
- const localRight = new Float32Array(localSamplesCount);
- const localChannels = [localLeft, localRight];
+ const bus = new MixBus(localSamplesCount);
const processSegment = async (
seg: { startSec: number; endSec?: number } | null,
@@ -574,54 +765,76 @@ export async function mixAudioTracks(
(Number(params.trimStartSec) || 0) + totalSeekSec,
);
- // Load and decode from cache or file
- let decoded = decodedCache.get(params.sourceUrl);
- if (decoded === undefined) {
- decoded = await decodeAudioSource(params.sourceUrl);
- decodedCache.set(params.sourceUrl, decoded);
- }
+ const clipDurationSec = actualDurationFrames / fps;
+ const clipSamplesCount = Math.min(
+ localSamplesCount - segmentStartSample,
+ Math.ceil(clipDurationSec * targetSampleRate),
+ );
+
+ // Decode (or reuse) only the stretch of the source this clip plays
+ const decoded =
+ clipSamplesCount > 0 && segmentStartSample < localSamplesCount
+ ? await getDecodedRange(
+ params.sourceUrl,
+ effectiveTrimSec,
+ effectiveTrimSec +
+ (clipSamplesCount / targetSampleRate) * finalSpeed +
+ RANGE_MARGIN_SEC,
+ )
+ : null;
if (decoded) {
- const clipDurationSec = actualDurationFrames / fps;
- const clipSamplesCount = Math.min(
- localSamplesCount - segmentStartSample,
- Math.ceil(clipDurationSec * targetSampleRate),
+ const trimOffsetSamples = Math.round(
+ effectiveTrimSec * decoded.sampleRate,
);
- if (
- clipSamplesCount > 0 &&
- segmentStartSample < localSamplesCount
- ) {
- const trimOffsetSamples = Math.round(
- effectiveTrimSec * decoded.sampleRate,
- );
-
- // Resample left & right channels
- const srcLeft = decoded.channels[0];
- const srcRight =
- decoded.channels.length > 1
- ? decoded.channels[1]
- : decoded.channels[0]; // fallback to mono
-
- const stretchedLeft = await resampleAndStretchPCM(
- srcLeft,
- decoded.sampleRate,
- targetSampleRate,
- finalSpeed,
- clipSamplesCount,
- trimOffsetSamples,
- );
-
- const stretchedRight = await resampleAndStretchPCM(
- srcRight,
- decoded.sampleRate,
- targetSampleRate,
- finalSpeed,
- clipSamplesCount,
- trimOffsetSamples,
- );
+ // Resample left & right channels
+ const stretchedLeft = await resampleAndStretchPCM(
+ decoded.channels[0],
+ decoded.sampleRate,
+ targetSampleRate,
+ finalSpeed,
+ clipSamplesCount,
+ trimOffsetSamples,
+ decoded.startIndex,
+ );
+ // A mono source plays the same samples on both sides
+ const stretchedRight =
+ decoded.channels.length > 1
+ ? await resampleAndStretchPCM(
+ decoded.channels[1],
+ decoded.sampleRate,
+ targetSampleRate,
+ finalSpeed,
+ clipSamplesCount,
+ trimOffsetSamples,
+ decoded.startIndex,
+ )
+ : stretchedLeft;
+
+ const stretched = [stretchedLeft, stretchedRight];
+ if (bus.canAdopt(stretched, segmentStartSample)) {
+ // The clip fills the silent bus: scale it in place and keep it
+ if (volume !== 1) {
+ for (let i = 0; i < clipSamplesCount; i++) {
+ stretchedLeft[i] *= volume;
+ }
+ if (stretchedRight !== stretchedLeft) {
+ for (let i = 0; i < clipSamplesCount; i++) {
+ stretchedRight[i] *= volume;
+ }
+ }
+ }
+ bus.channels = [
+ stretchedLeft,
+ stretchedRight === stretchedLeft
+ ? stretchedLeft.slice()
+ : stretchedRight,
+ ];
+ } else {
// Add to local mix with volume scaling
+ const [localLeft, localRight] = bus.get();
for (let i = 0; i < clipSamplesCount; i++) {
const outIndex = segmentStartSample + i;
localLeft[outIndex] += stretchedLeft[i] * volume;
@@ -651,17 +864,7 @@ export async function mixAudioTracks(
const childStartSample = Math.round(
(childStartFrame / fps) * targetSampleRate,
);
- const destStartSample = segmentStartSample + childStartSample;
- for (let c = 0; c < 2; c++) {
- const dest = localChannels[c];
- const src = childChannels[c];
- const len = Math.min(src.length, dest.length - destStartSample);
- for (let i = 0; i < len; i++) {
- if (destStartSample + i >= 0) {
- dest[destStartSample + i] += src[i];
- }
- }
- }
+ bus.add(childChannels, segmentStartSample + childStartSample);
}
}
}
@@ -681,12 +884,14 @@ export async function mixAudioTracks(
segmentStartSample < localSamplesCount &&
segmentStartSample >= 0
) {
- const slicedChannels = localChannels.map((ch) =>
- ch.subarray(
- segmentStartSample,
- segmentStartSample + clipSamplesCount,
- ),
- );
+ const slicedChannels = bus
+ .get()
+ .map((ch) =>
+ ch.subarray(
+ segmentStartSample,
+ segmentStartSample + clipSamplesCount,
+ ),
+ );
// The layer's own clock: its in-point moves the source, not its
// animation, and a CompositorLayer's keyframes count from its
// own first frame (as its picture's do), not the composition's.
@@ -729,12 +934,23 @@ export async function mixAudioTracks(
await processSegment(null, 0);
}
- return localChannels;
+ return bus.channels;
}
const normalized = normalizeTimeline(virtualMedia, fps);
const rootChannels = await traverse(normalized, 0, 0, 1, 1, null);
- if (rootChannels) {
+
+ // Output mixed buffer is stereo; a root mix of the same length is used as is
+ const mixedChannels =
+ rootChannels?.every((ch) => ch.length === totalTargetSamples) &&
+ rootChannels[0] !== rootChannels[1]
+ ? rootChannels
+ : [
+ new Float32Array(totalTargetSamples),
+ new Float32Array(totalTargetSamples),
+ ];
+ const [outputLeft, outputRight] = mixedChannels;
+ if (rootChannels && mixedChannels !== rootChannels) {
for (let c = 0; c < 2; c++) {
const dest = mixedChannels[c];
const src = rootChannels[c];
diff --git a/packages/core/src/signals/index.ts b/packages/core/src/signals/index.ts
index dce89e78..6494153a 100644
--- a/packages/core/src/signals/index.ts
+++ b/packages/core/src/signals/index.ts
@@ -316,6 +316,14 @@ export function signal<T>(initialValue: T): WritableSignal<T> {
return new StateSignalImpl<T>(initialValue);
}
+/**
+ * Whether `val` is a plain value signal (from `signal()`): it reads the same
+ * whatever frame context it is given, so one read stands for every frame.
+ */
+export function isValueSignal(val: unknown): val is WritableSignal<unknown> {
+ return val instanceof StateSignalImpl;
+}
+
export function computed<T>(fn: () => T): Signal<T> {
return new ComputedImpl<T>(fn);
}
diff --git a/packages/core/src/signals/signals.test.ts b/packages/core/src/signals/signals.test.ts
index 15d796d4..cd2d3472 100644
--- a/packages/core/src/signals/signals.test.ts
+++ b/packages/core/src/signals/signals.test.ts
@@ -4,6 +4,7 @@ import {
frameArraySignal,
frameSignal,
isSignal,
+ isValueSignal,
Signal,
signal,
updateClockSignals,
@@ -174,4 +175,12 @@ describe("Signals Reactive Core", () => {
});
expect(valAtHalfSec).toBe(20);
});
+
+ it("tells plain value signals from frame-dependent ones", () => {
+ expect(isValueSignal(signal(0.5))).toBe(true);
+ expect(isValueSignal(computed(() => 1))).toBe(false);
+ expect(isValueSignal(Signal.programmatic((ctx) => ctx.time))).toBe(false);
+ expect(isValueSignal(frameArraySignal([1, 2, 3]))).toBe(false);
+ expect(isValueSignal(0.5)).toBe(false);
+ });
});
diff --git a/packages/framefields/src/audio-mix-ranges.test.ts b/packages/framefields/src/audio-mix-ranges.test.ts
new file mode 100644
index 00000000..985d17e3
--- /dev/null
+++ b/packages/framefields/src/audio-mix-ranges.test.ts
@@ -0,0 +1,151 @@
+/**
+ * The mix decodes only the stretch of a source each clip plays and skips
+ * silent layers. Both must leave the mix sample-for-sample what a whole-file
+ * decode gives, including for compressed sources decoded from mid-file.
+ */
+import { execFileSync } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { decodeAudioSource } from "@framefields/compositions";
+import { beforeAll, describe, expect, it } from "vitest";
+import { Composition, encodeStereoWav, Layer } from "./index.js";
+import { HeadlessMediaRenderer } from "./renderer/index.js";
+
+const SR = 48000;
+const FPS = 30;
+const SOURCE_SEC = 40;
+const TRIM_SEC = 31.5;
+
+const hasFfmpeg = (() => {
+ try {
+ execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });
+ return true;
+ } catch {
+ return false;
+ }
+})();
+
+/** Seeded noise, distinct per channel, quiet enough that no codec peak trips the mix's auto-gain. */
+function noise(seconds: number, seed: number): Float32Array {
+ const out = new Float32Array(Math.round(seconds * SR));
+ let s = seed;
+ for (let i = 0; i < out.length; i++) {
+ s = (s * 1103515245 + 12345) >>> 0;
+ out[i] = ((s / 2 ** 32) * 2 - 1) * 0.3;
+ }
+ return out;
+}
+
+let dir = "";
+const sources: Record<string, string> = {};
+
+beforeAll(async () => {
+ dir = await fs.mkdtemp(path.join(os.tmpdir(), "audio-mix-ranges-"));
+ sources.wav = path.join(dir, "source.wav");
+ await fs.writeFile(
+ sources.wav,
+ encodeStereoWav([noise(SOURCE_SEC, 1), noise(SOURCE_SEC, 2)], {
+ sampleRate: SR,
+ }),
+ );
+ if (hasFfmpeg) {
+ for (const [ext, codec] of [
+ ["mp3", "libmp3lame"],
+ ["m4a", "aac"],
+ ]) {
+ sources[ext] = path.join(dir, `source.${ext}`);
+ execFileSync("ffmpeg", [
+ ...["-v", "error", "-y", "-i", sources.wav],
+ ...["-c:a", codec, "-b:a", "128k", sources[ext]],
+ ]);
+ }
+ }
+}, 120_000);
+
+async function mix(comp: Composition) {
+ return new HeadlessMediaRenderer().renderAudio(comp.toVirtualMedia(), {
+ fps: FPS,
+ sampleRate: SR,
+ });
+}
+
+describe("audio mix source ranges", () => {
+ for (const ext of ["wav", "mp3", "m4a"]) {
+ it.skipIf(ext !== "wav" && !hasFfmpeg)(
+ `plays a clip late in a long ${ext} source as the whole file decodes it`,
+ async () => {
+ const comp = new Composition({
+ width: 64,
+ height: 64,
+ fps: FPS,
+ durationMs: 3000,
+ });
+ comp.addAudio(
+ Layer.audio(sources[ext], {
+ trimStartSec: TRIM_SEC,
+ startFrame: 15,
+ durationFrames: 60,
+ }),
+ );
+ const { channels } = await mix(comp);
+
+ const whole = await decodeAudioSource(sources[ext]);
+ expect(whole?.sampleRate).toBe(SR);
+ const from = Math.round(TRIM_SEC * SR);
+ const start = Math.round((15 / FPS) * SR);
+ const length = Math.round((60 / FPS) * SR);
+ for (let c = 0; c < 2; c++) {
+ const out = channels[c];
+ const src = whole?.channels[c] as Float32Array;
+ let mismatches = 0;
+ for (let i = 0; i < length; i++) {
+ // The mix's soft limiter is a plain tanh at this level
+ if (out[start + i] !== Math.fround(Math.tanh(src[from + i]))) {
+ mismatches++;
+ }
+ }
+ expect(mismatches).toBe(0);
+ expect(out[start - 1]).toBe(0);
+ expect(out[start + length]).toBe(0);
+ }
+ },
+ 60_000,
+ );
+ }
+
+ it("leaves the mix unchanged by silent visual layers", async () => {
+ const build = (withVisuals: boolean) => {
+ const comp = new Composition({
+ width: 64,
+ height: 64,
+ fps: FPS,
+ durationMs: 2000,
+ });
+ if (withVisuals) {
+ for (let i = 0; i < 4; i++) comp.add(Layer.text(`t${i}`));
+ }
+ comp.addAudio(
+ Layer.audio(sources.wav, { trimStartSec: 3, durationFrames: 45 }),
+ );
+ comp.addAudio(
+ Layer.audio(sources.wav, {
+ trimStartSec: 20,
+ startFrame: 30,
+ volume: 0.5,
+ }),
+ );
+ return comp;
+ };
+ const plain = await mix(build(false));
+ const withVisuals = await mix(build(true));
+ for (let c = 0; c < 2; c++) {
+ expect(withVisuals.channels[c]).toEqual(plain.channels[c]);
+ }
+ // Both clips are heard where they play
+ expect(Math.abs(plain.channels[0][SR / 2])).toBeGreaterThan(0);
+ expect(Math.abs(plain.channels[0][Math.round(1.5 * SR)])).toBeGreaterThan(
+ 0,
+ );
+ }, 60_000);
+});
diff --git a/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts b/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts
index 594386f2..24279329 100644
--- a/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts
+++ b/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts
@@ -1,4 +1,8 @@
-import { isSignal, type VirtualMediaData } from "@framefields/core";
+import {
+ isSignal,
+ isValueSignal,
+ type VirtualMediaData,
+} from "@framefields/core";
import { shaderStore } from "../shader-store.js";
import { signalRegistry } from "../signals/signal-registry.js";
@@ -284,22 +288,27 @@ export class WebGPUAudioProcessor {
const signalData = new Float32Array(numSamples);
const durationSec =
durationSeconds > 0 ? durationSeconds : numSamples / _sampleRate;
- for (let s = 0; s < numSamples; s++) {
- const t = elapsedSeconds + s / _sampleRate;
- const f = t * fps;
- const progress = durationSec > 0 ? t / durationSec : 0;
- signalData[s] =
- Number(
- directVal.get({
- frame: f,
- fps,
- time: t,
- duration: durationSec * fps,
- durationMs: durationSec * 1000,
- progress,
- deltaTime: 1 / _sampleRate,
- }),
- ) || 0;
+ if (isValueSignal(directVal)) {
+ // A plain value reads the same at every sample: read it once
+ signalData.fill(Number(directVal.get()) || 0);
+ } else {
+ for (let s = 0; s < numSamples; s++) {
+ const t = elapsedSeconds + s / _sampleRate;
+ const f = t * fps;
+ const progress = durationSec > 0 ? t / durationSec : 0;
+ signalData[s] =
+ Number(
+ directVal.get({
+ frame: f,
+ fps,
+ time: t,
+ duration: durationSec * fps,
+ durationMs: durationSec * 1000,
+ progress,
+ deltaTime: 1 / _sampleRate,
+ }),
+ ) || 0;
+ }
}
const buf = device.createBuffer({
size: Math.max(16, Math.ceil(signalData.byteLength / 16) * 16),
@@ -400,6 +409,35 @@ export class WebGPUAudioProcessor {
// 3. Process in chunks to prevent GPU driver timeout (TDR) and browser freezing
const CHUNK_SIZE = 65536;
+ // Chunks submitted before the oldest is read back. The queue runs them in
+ // order (the state buffer carries across), so waiting on each readback
+ // before submitting the next only adds a GPU round trip per chunk.
+ const MAX_CHUNKS_IN_FLIGHT = 8;
+ const pending: {
+ stagingBuffer: GPUBuffer;
+ offset: number;
+ chunkSamples: number;
+ }[] = [];
+
+ const readBackOldest = async () => {
+ const chunk = pending.shift();
+ if (!chunk) return;
+ const { stagingBuffer, offset, chunkSamples } = chunk;
+ await stagingBuffer.mapAsync(GPUMapMode.READ);
+ const mappedRange = new Float32Array(stagingBuffer.getMappedRange());
+
+ for (let c = 0; c < numChannels; c++) {
+ channels[c].set(
+ mappedRange.subarray(c * chunkSamples, (c + 1) * chunkSamples),
+ offset,
+ );
+ }
+
+ stagingBuffer.unmap();
+ stagingBuffer.destroy();
+
+ await yieldToMain();
+ };
try {
for (let offset = 0; offset < numSamples; offset += CHUNK_SIZE) {
@@ -572,22 +610,19 @@ export class WebGPUAudioProcessor {
device.queue.submit([commandEncoder.finish()]);
- // Map & Read back
- await stagingBuffer.mapAsync(GPUMapMode.READ);
- const mappedRange = new Float32Array(stagingBuffer.getMappedRange());
-
- for (let c = 0; c < numChannels; c++) {
- channels[c].set(
- mappedRange.subarray(c * chunkSamples, (c + 1) * chunkSamples),
- offset,
- );
+ // Read back later; the next chunk's writes queue behind this one
+ pending.push({ stagingBuffer, offset, chunkSamples });
+ if (pending.length >= MAX_CHUNKS_IN_FLIGHT) {
+ await readBackOldest();
}
-
- stagingBuffer.unmap();
-
- await yieldToMain();
+ }
+ while (pending.length > 0) {
+ await readBackOldest();
}
} finally {
+ for (const { stagingBuffer } of pending) {
+ stagingBuffer.destroy();
+ }
for (const b of createdBuffers) {
b.destroy();
}